diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index b5aa4e747..000000000 --- a/.gitattributes +++ /dev/null @@ -1,3 +0,0 @@ -*.json filter=lfs diff=lfs merge=lfs -text -tests/unit/metrics/test_cases/*.json -filter -diff -merge text -*.parquet filter=lfs diff=lfs merge=lfs -text diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 1a8c26cfd..000000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,20 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve lighteval! -title: "[BUG] " -labels: bug -assignees: '' - ---- - -## Describe the bug -A clear and concise description of what the bug is. - -## To Reproduce -Please provide all the steps needed to reproduce the behavior, or provide a minimal working example if needed. We will ignore issues missing this section. - -## Expected behavior -A clear and concise description of what you expected to happen. - -## Version info -Please provide your operating system, lighteval version or commit if you installed from main, and pip/conda environment if your problem concerns dependencies. diff --git a/.github/ISSUE_TEMPLATE/evaluation-task-request.md b/.github/ISSUE_TEMPLATE/evaluation-task-request.md deleted file mode 100644 index 4b890858f..000000000 --- a/.github/ISSUE_TEMPLATE/evaluation-task-request.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -name: Evaluation/task request -about: Suggest a new evaluation you want us to add -title: "[EVAL]" -labels: new task -assignees: '' - ---- - -## Evaluation short description -- Why is this evaluation interesting? -- How used is it in the community? - -## Evaluation metadata -Provide all available -- Paper url: -- Github url: -- Dataset url: diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md deleted file mode 100644 index de215e72b..000000000 --- a/.github/ISSUE_TEMPLATE/feature-request.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project -title: "[FT] " -labels: feature request -assignees: '' - ---- - -## Issue encountered -Is your feature request related to a problem? Please provide a clear and concise description of what the problem is. Ex. I'm always frustrated when [...] - -## Solution/Feature -A clear and concise description of what you want to happen. - -## Possible alternatives -A clear and concise description of any alternative solutions or features you've considered. diff --git a/.github/release.yml b/.github/release.yml deleted file mode 100644 index 6a6eec3ed..000000000 --- a/.github/release.yml +++ /dev/null @@ -1,26 +0,0 @@ -changelog: - exclude: - labels: - - ignore-for-release - categories: - - title: New Features 🎉 - labels: - - feature - - title: Enhancement ⚙️ - labels: - - enhancement - - title: Documentation 📚 - labels: - - documentation - - title: New Tasks - labels: - - new-task - - title: Task and Metrics changes 🛠️ - labels: - - task-update - - title: Bug Fixes 🐛 - labels: - - bug - - title: Other Changes - labels: - - "*" diff --git a/.github/workflows/doc-build.yml b/.github/workflows/doc-build.yml deleted file mode 100644 index cd345d3d3..000000000 --- a/.github/workflows/doc-build.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: Build Documentation - -on: - push: - branches: - - main - - doc-builder* - - v*-release - -jobs: - build: - uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@main - with: - commit_sha: ${{ github.sha }} - package: lighteval - secrets: - token: ${{ secrets.HUGGINGFACE_PUSH }} - hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }} diff --git a/.github/workflows/doc-pr-build.yml b/.github/workflows/doc-pr-build.yml deleted file mode 100644 index f96e20583..000000000 --- a/.github/workflows/doc-pr-build.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Build PR Documentation - -on: - pull_request: - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - build: - uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@main - with: - commit_sha: ${{ github.event.pull_request.head.sha }} - pr_number: ${{ github.event.number }} - package: lighteval diff --git a/.github/workflows/doc-pr-upload.yml b/.github/workflows/doc-pr-upload.yml deleted file mode 100644 index b4304ca07..000000000 --- a/.github/workflows/doc-pr-upload.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Upload PR Documentation - -on: - workflow_run: - workflows: ["Build PR Documentation"] - types: - - completed - -jobs: - build: - uses: huggingface/doc-builder/.github/workflows/upload_pr_documentation.yml@main - with: - package_name: lighteval - secrets: - hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }} - comment_bot_app_id: ${{ secrets.COMMENT_BOT_APP_ID }} - comment_bot_secret_pem: ${{ secrets.COMMENT_BOT_SECRET_PEM }} diff --git a/.github/workflows/pr_style_bot.yaml b/.github/workflows/pr_style_bot.yaml deleted file mode 100644 index e08643d83..000000000 --- a/.github/workflows/pr_style_bot.yaml +++ /dev/null @@ -1,16 +0,0 @@ -name: PR Style Bot - -on: - issue_comment: - types: [created] - -permissions: - pull-requests: write - -jobs: - style: - uses: huggingface/huggingface_hub/.github/workflows/style-bot-action.yml@main - with: - python_quality_dependencies: "[quality]" - secrets: - bot_token: ${{ secrets.HF_STYLE_BOT_ACTION }} diff --git a/.github/workflows/quality.yaml b/.github/workflows/quality.yaml deleted file mode 100644 index fa1490432..000000000 --- a/.github/workflows/quality.yaml +++ /dev/null @@ -1,30 +0,0 @@ -name: Quality - -on: - push: - branches: - - main - - v*-release - pull_request: - branches: - - main - -jobs: - - check_code_quality: - name: Check code quality - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - - name: Setup Python environment - uses: actions/setup-python@v2 - with: - python-version: '3.10' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install ".[quality]" - - name: Code quality - run: | - make quality diff --git a/.github/workflows/slow_tests.yaml b/.github/workflows/slow_tests.yaml deleted file mode 100644 index a1f68f9d4..000000000 --- a/.github/workflows/slow_tests.yaml +++ /dev/null @@ -1,45 +0,0 @@ -name: Slow end to end tests - -on: - push: - branches: - - main - - v*-release - pull_request: - branches: - - main - -jobs: - run_tests: - name: Run tests - runs-on: 'aws-g4dn-2xlarge-use1-public-80' - steps: - - name: Install Git LFS - run: | - if ! command -v git-lfs &> /dev/null; then - echo "Installing Git LFS..." - sudo apt-get update && sudo apt-get install -y git-lfs - git lfs install - else - echo "Git LFS already installed." - fi - - - name: Checkout repository - uses: actions/checkout@v4 - with: - lfs: true - - - name: Install uv - uses: astral-sh/setup-uv@v5 - with: - enable-cache: true - - - name: Install the project - run: uv sync --extra dev - - - - name: run nvidia-smi - run: nvidia-smi - - - name: Run tests - run: uv run pytest --disable-pytest-warnings --runslow tests/slow_tests/ diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml deleted file mode 100644 index e1a2cad51..000000000 --- a/.github/workflows/tests.yaml +++ /dev/null @@ -1,59 +0,0 @@ -name: Tests - -on: - push: - branches: - - main - - v*-release - pull_request: - branches: - - main - -jobs: - run_tests: - name: Run tests - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - lfs: true - - - name: Cache Hugging Face models - uses: actions/cache@v4 - with: - path: cache/models - key: hf-models-${{ runner.os }}-${{ github.ref }} - restore-keys: hf-models-${{ runner.os }}- - - - name: Cache Hugging Face datasets - uses: actions/cache@v4 - with: - path: cache/datasets - key: hf-datasets-${{ runner.os }}-${{ github.ref }} - restore-keys: hf-datasets-${{ runner.os }}- - - - name: Cache uv virtual environment - uses: actions/cache@v4 - with: - path: .venv - key: uv-env-${{ runner.os }}-${{ hashFiles('pyproject.toml') }} - restore-keys: uv-env-${{ runner.os }}- - - - name: Install uv - uses: astral-sh/setup-uv@v5 - with: - enable-cache: true - - - name: Install the project - run: uv sync --extra dev - - - name: Ensure cache directories exist - run: mkdir -p cache/models cache/datasets - - - name: Run tests - env: - HF_TEST_TOKEN: ${{ secrets.HF_TEST_TOKEN }} - HF_HOME: "cache/models" - HF_DATASETS_CACHE: "cache/datasets" - run: uv run pytest -x --disable-pytest-warnings diff --git a/.github/workflows/trufflehog.yml b/.github/workflows/trufflehog.yml deleted file mode 100644 index 5deb937e5..000000000 --- a/.github/workflows/trufflehog.yml +++ /dev/null @@ -1,20 +0,0 @@ -on: - push: - -name: Scan Secret Leaks - -permissions: - contents: read - -jobs: - trufflehog: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Secret Scanning - uses: trufflesecurity/trufflehog@main - with: - extra_args: --only-verified diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 77e347cf5..000000000 --- a/.gitignore +++ /dev/null @@ -1,185 +0,0 @@ -# log and data files -*.model -*.pkl -#*.ipynb -output -result -*.pt -tests/data/asr -.DS_Store -bert.pt.json -work -runs -*.parquet -*.json -cache/ -fastspeech_output -.hydra -.bash_history.local - -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class -**.pyc - -# C extensions -*.so - -# Distribution / packaging -.idea -.Python -wandb -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -#parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/build - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# Override Jupyter in Github Language states for more accurate estimate of repo code. -# Reference: https://github.com/github/linguist/blob/master/docs/overrides.md#generated-code -*.ipynb linguist-generated - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don’t work, or not -# install all needed dependencies. -#Pipfile.lock - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# VSCode project settins -.vscode/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site -/docs/html -/docs/docs_zh/zh - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# Emacs backup files -*~ - -cifar-10-batches-py -*.tar.gz - -# Test data. -tests/.data -tests/data - -# outputs folder -wandb -dump.py - -docs/sources/source/test_build/ - -# Checkpoints, config files and temporary files created in tutorials. -.hydra/ -nemo_experiments/ - -.ruff_cache - -tmp.py - -benchmark_output -prod_env - -logs -_logs -outputs diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index b832c5a5c..000000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -default_language_version: - python: python3 - -ci: - autofix_prs: true - autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions' - autoupdate_schedule: quarterly - -repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.3.0 - hooks: - - id: check-yaml - - id: check-case-conflict - - id: detect-private-key - - id: check-added-large-files - args: ['--maxkb=1000'] - - id: end-of-file-fixer - - id: trailing-whitespace - - - repo: https://github.com/charliermarsh/ruff-pre-commit - # Ruff version. - rev: 'v0.11.10' - hooks: - - id: ruff - args: ['--fix'] - - id: ruff-format diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 5e9693e35..000000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2024 Hugging Face - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index f31276637..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -include src/lighteval/tasks/tasks_table.jsonl -include src/lighteval/metrics/*.jsonl diff --git a/Makefile b/Makefile deleted file mode 100644 index e50483f1e..000000000 --- a/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -.PHONY: style format - - -style: - ruff format . - ruff check --fix . - - -quality: - ruff format --check . - ruff check . diff --git a/README.md b/README.md deleted file mode 100644 index 72473a34a..000000000 --- a/README.md +++ /dev/null @@ -1,204 +0,0 @@ -

-
- lighteval library logo -
-

- - -

- Your go-to toolkit for lightning-fast, flexible LLM evaluation, from Hugging Face's Leaderboard and Evals Team. -

- -
- -[![Tests](https://github.com/huggingface/lighteval/actions/workflows/tests.yaml/badge.svg?branch=main)](https://github.com/huggingface/lighteval/actions/workflows/tests.yaml?query=branch%3Amain) -[![Quality](https://github.com/huggingface/lighteval/actions/workflows/quality.yaml/badge.svg?branch=main)](https://github.com/huggingface/lighteval/actions/workflows/quality.yaml?query=branch%3Amain) -[![Python versions](https://img.shields.io/pypi/pyversions/lighteval)](https://www.python.org/downloads/) -[![License](https://img.shields.io/badge/License-MIT-green.svg)](https://github.com/huggingface/lighteval/blob/main/LICENSE) -[![Version](https://img.shields.io/pypi/v/lighteval)](https://pypi.org/project/lighteval/) - -
- ---- - -

- - Documentation - - - Open Benchmark Index - -

- ---- - -**Lighteval** is your *all-in-one toolkit* for evaluating LLMs across multiple -backends—whether your model is being **served somewhere** or **already loaded in memory**. -Dive deep into your model's performance by saving and exploring *detailed, -sample-by-sample results* to debug and see how your models stack-up. - -*Customization at your fingertips*: letting you either browse all our existing tasks and [metrics](https://huggingface.co/docs/lighteval/metric-list) or effortlessly create your own [custom task](https://huggingface.co/docs/lighteval/adding-a-custom-task) and [custom metric](https://huggingface.co/docs/lighteval/adding-a-new-metric), tailored to your needs. - - -## Available Tasks - -Lighteval supports **1000+ evaluation tasks** across multiple domains and -languages. Use [this -space](https://huggingface.co/spaces/OpenEvals/open_benchmark_index) to find what -you need, or, here's an overview of some *popular benchmarks*: - - -### 📚 **Knowledge** -- **General Knowledge**: MMLU, MMLU-Pro, MMMU, BIG-Bench -- **Question Answering**: TriviaQA, Natural Questions, SimpleQA, Humanity's Last Exam (HLE) -- **Specialized**: GPQA, AGIEval - -### 🧮 **Math and Code** -- **Math Problems**: GSM8K, GSM-Plus, MATH, MATH500 -- **Competition Math**: AIME24, AIME25 -- **Multilingual Math**: MGSM (Grade School Math in 10+ languages) -- **Coding Benchmarks**: LCB (LiveCodeBench) - -### 🎯 **Chat Model Evaluation** -- **Instruction Following**: IFEval, IFEval-fr -- **Reasoning**: MUSR, DROP (discrete reasoning) -- **Long Context**: RULER -- **Dialogue**: MT-Bench -- **Holistic Evaluation**: HELM, BIG-Bench - -### 🌍 **Multilingual Evaluation** -- **Cross-lingual**: XTREME, Flores200 (200 languages), XCOPA, XQuAD -- **Language-specific**: - - **Arabic**: ArabicMMLU - - **Filipino**: FilBench - - **French**: IFEval-fr, GPQA-fr, BAC-fr - - **German**: German RAG Eval - - **Serbian**: Serbian LLM Benchmark, OZ Eval - - **Turkic**: TUMLU (9 Turkic languages) - - **Chinese**: CMMLU, CEval, AGIEval - - **Russian**: RUMMLU, Russian SQuAD - - **Kyrgyz**: Kyrgyz LLM Benchmark - - **And many more...** - -### 🧠 **Core Language Understanding** -- **NLU**: GLUE, SuperGLUE, TriviaQA, Natural Questions -- **Commonsense**: HellaSwag, WinoGrande, ProtoQA -- **Natural Language Inference**: XNLI -- **Reading Comprehension**: SQuAD, XQuAD, MLQA, Belebele - - -## ⚡️ Installation - -> **Note**: lighteval is currently *completely untested on Windows*, and we don't support it yet. (*Should be fully functional on Mac/Linux*) - -```bash -pip install lighteval -``` - -Lighteval allows for *many extras* when installing, see [here](https://huggingface.co/docs/lighteval/installation) for a **complete list**. - -If you want to push results to the **Hugging Face Hub**, add your access token as -an environment variable: - -```shell -hf auth login -``` - -## 🚀 Quickstart - -Lighteval offers the following entry points for model evaluation: - -- `lighteval eval`: Evaluation models using [inspect-ai](https://inspect.aisi.org.uk/) as a backend (prefered). -- `lighteval accelerate`: Evaluate models on CPU or one or more GPUs using [🤗 - Accelerate](https://github.com/huggingface/accelerate) -- `lighteval nanotron`: Evaluate models in distributed settings using [⚡️ - Nanotron](https://github.com/huggingface/nanotron) -- `lighteval vllm`: Evaluate models on one or more GPUs using [🚀 - VLLM](https://github.com/vllm-project/vllm) -- `lighteval sglang`: Evaluate models using [SGLang](https://github.com/sgl-project/sglang) as backend -- `lighteval endpoint`: Evaluate models using various endpoints as backend - - `lighteval endpoint inference-endpoint`: Evaluate models using Hugging Face's [Inference Endpoints API](https://huggingface.co/inference-endpoints/dedicated) - - `lighteval endpoint tgi`: Evaluate models using [🔗 Text Generation Inference](https://huggingface.co/docs/text-generation-inference/en/index) running locally - - `lighteval endpoint litellm`: Evaluate models on any compatible API using [LiteLLM](https://www.litellm.ai/) - - `lighteval endpoint inference-providers`: Evaluate models using [HuggingFace's inference providers](https://huggingface.co/docs/inference-providers/en/index) as backend - -Did not find what you need ? You can always make your custom model API by following [this guide](https://huggingface.co/docs/lighteval/main/en/evaluating-a-custom-model) -- `lighteval custom`: Evaluate custom models (can be anything) - -Here's a **quick command** to evaluate using a remote inference service: - -```shell -lighteval eval "hf-inference-providers/openai/gpt-oss-20b" gpqa:diamond -``` - -Or use the **Python API** to run a model *already loaded in memory*! - -```python -from transformers import AutoModelForCausalLM - -from lighteval.logging.evaluation_tracker import EvaluationTracker -from lighteval.models.transformers.transformers_model import TransformersModel, TransformersModelConfig -from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters - - -MODEL_NAME = "meta-llama/Meta-Llama-3-8B-Instruct" -BENCHMARKS = "gsm8k" - -evaluation_tracker = EvaluationTracker(output_dir="./results") -pipeline_params = PipelineParameters( - launcher_type=ParallelismManager.NONE, - max_samples=2 -) - -model = AutoModelForCausalLM.from_pretrained( - MODEL_NAME, device_map="auto" -) -config = TransformersModelConfig(model_name=MODEL_NAME, batch_size=1) -model = TransformersModel.from_model(model, config) - -pipeline = Pipeline( - model=model, - pipeline_parameters=pipeline_params, - evaluation_tracker=evaluation_tracker, - tasks=BENCHMARKS, -) - -results = pipeline.evaluate() -pipeline.show_results() -results = pipeline.get_results() -``` - -## 🙏 Acknowledgements - -Lighteval took inspiration from the following *amazing* frameworks: Eleuther's [AI Harness](https://github.com/EleutherAI/lm-evaluation-harness) and Stanford's -[HELM](https://crfm.stanford.edu/helm/latest/). We are grateful to their teams for their **pioneering work** on LLM evaluations. - -We'd also like to offer our thanks to all the community members who have contributed to the library, adding new features and reporting or fixing bugs. - -## 🌟 Contributions Welcome 💙💚💛💜🧡 - -**Got ideas?** Found a bug? Want to add a -[task](https://huggingface.co/docs/lighteval/adding-a-custom-task) or -[metric](https://huggingface.co/docs/lighteval/adding-a-new-metric)? -Contributions are *warmly welcomed*! - -If you're adding a **new feature**, please *open an issue first*. - -If you open a PR, don't forget to **run the styling**! - -```bash -pip install -e ".[dev]" -pre-commit install -pre-commit run --all-files -``` -## 📜 Citation - -```bibtex -@misc{lighteval, - author = {Habib, Nathan and Fourrier, Clémentine and Kydlíček, Hynek and Wolf, Thomas and Tunstall, Lewis}, - title = {LightEval: A lightweight framework for LLM evaluation}, - year = {2023}, - version = {0.11.0}, - url = {https://github.com/huggingface/lighteval} -} -``` diff --git a/assets/lighteval-doc.svg b/assets/lighteval-doc.svg deleted file mode 100644 index 4311f6ca4..000000000 --- a/assets/lighteval-doc.svg +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml deleted file mode 100644 index 75733d49b..000000000 --- a/docs/source/_toctree.yml +++ /dev/null @@ -1,64 +0,0 @@ -- sections: - - local: index - title: 🤗 Lighteval - - local: installation - title: Installation - - local: quicktour - title: Quicktour - title: Getting started -- sections: - - local: inspect-ai - title: Examples using Inspect-AI - - local: saving-and-reading-results - title: Save and read results - - local: caching - title: Caching - - local: using-the-python-api - title: Use the Python API - - local: adding-a-custom-task - title: Add a custom task - - local: offline-evaluation - title: Offline evaluation - - local: adding-a-new-metric - title: Add a custom metric - - local: evaluating-a-custom-model - title: Evaluate a custom model - - local: use-inference-providers-as-backend - title: Use HF's inference providers as backend - - local: use-litellm-as-backend - title: Use litellm as backend - - local: use-vllm-as-backend - title: Use vllm as backend - - local: use-sglang-as-backend - title: Use SGLang as backend - - local: use-huggingface-inference-endpoints-or-tgi-as-backend - title: Use Hugging Face inference endpoints or TGI as backend - - local: contributing-to-multilingual-evaluations - title: Contributing to multilingual evaluations - title: Guides -- sections: - - local: metric-list - title: Available Metrics - - local: available-tasks - title: Available Tasks - title: API -- sections: - - sections: - - local: package_reference/evaluation_tracker - title: EvaluationTracker - - local: package_reference/models - title: Model Configs - - local: package_reference/pipeline - title: Pipeline - title: Main classes - - local: package_reference/metrics - title: Metrics - - local: package_reference/tasks - title: Tasks - - local: package_reference/logging - title: Logging - - local: package_reference/models_outputs - title: ModelResponse - - local: package_reference/doc - title: Doc - title: Reference diff --git a/docs/source/adding-a-custom-task.mdx b/docs/source/adding-a-custom-task.mdx deleted file mode 100644 index c68149a1e..000000000 --- a/docs/source/adding-a-custom-task.mdx +++ /dev/null @@ -1,162 +0,0 @@ -# Adding a Custom Task - -Lighteval provides a flexible framework for creating custom evaluation tasks. This guide explains how to create and integrate new tasks into the evaluation system. - -## Step-by-Step Creation of a Task - -> [!WARNING] -> To contribute your task to the Lighteval repository, you would first need -> to install the required dev dependencies by running `pip install -e .[dev]` -> and then run `pre-commit install` to install the pre-commit hooks. - -### Step 1: Create the Task File - -First, create a Python file or directory under the `src/lighteval/tasks/tasks` directory. -A directory is helpfull if you need to split your file into multiple ones, just make sure to have one of the file named `main.py`. - -### Step 2: Define the Prompt Function - -You need to define a prompt function that will convert a line from your -dataset to a document to be used for evaluation. - -```python -from lighteval.tasks.requests import Doc - -# Define as many as you need for your different tasks -def prompt_fn(line: dict, task_name: str): - """Defines how to go from a dataset line to a doc object. - Follow examples in src/lighteval/tasks/default_prompts.py, or get more info - about what this function should do in the README. - """ - return Doc( - task_name=task_name, - query=line["question"], - choices=[f" {c}" for c in line["choices"]], - gold_index=line["gold"], - ) -``` - -### Step 3: Choose or Create Metrics - -You can either use an existing metric (defined in [`lighteval.metrics.metrics.Metrics`]) or [create a custom one](adding-a-new-metric). - -#### Using Existing Metrics - -```python -from lighteval.metrics import Metrics - -# Use an existing metric -metric = Metrics.ACCURACY -``` - -#### Creating Custom Metrics - -```python -from lighteval.metrics.utils.metric_utils import SampleLevelMetric -import numpy as np - -custom_metric = SampleLevelMetric( - metric_name="my_custom_metric_name", - higher_is_better=True, - category="accuracy", - sample_level_fn=lambda x: x, # How to compute score for one sample - corpus_level_fn=np.mean, # How to aggregate the sample metrics -) -``` - -### Step 4: Define Your Task - -You can define a task with or without subsets using [`~tasks.lighteval_task.LightevalTaskConfig`]. - -#### Simple Task (No Subsets) - -```python -from lighteval.tasks.lighteval_task import LightevalTaskConfig - -# This is how you create a simple task (like HellaSwag) which has one single subset -# attached to it, and one evaluation possible. -task = LightevalTaskConfig( - name="myothertask", - prompt_function=prompt_fn, # Must be defined in the file or imported - hf_repo="your_dataset_repo_on_hf", - hf_subset="default", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split="train", - few_shots_select="random_sampling_from_train", - metrics=[metric], # Select your metric in Metrics - generation_size=256, - stop_sequence=["\n", "Question:"], -) -``` - -#### Task with Multiple Subsets - -If you want to create a task with multiple subsets, add them to the -`SAMPLE_SUBSETS` list and create a task for each subset. - -```python -SAMPLE_SUBSETS = ["subset1", "subset2", "subset3"] # List of all the subsets to use for this eval - -class CustomSubsetTask(LightevalTaskConfig): - def __init__( - self, - name, - hf_subset, - ): - super().__init__( - name=name, - hf_subset=hf_subset, - prompt_function=prompt_fn, # Must be defined in the file or imported - hf_repo="your_dataset_name", - metrics=[custom_metric], # Select your metric in Metrics or use your custom_metric - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split="train", - few_shots_select="random_sampling_from_train", - generation_size=256, - stop_sequence=["\n", "Question:"], - ) - -SUBSET_TASKS = [CustomSubsetTask(name=f"task:{subset}", hf_subset=subset) for subset in SAMPLE_SUBSETS] -``` - -### Step 5: Add Tasks to the Table - -Then you need to add your task to the `TASKS_TABLE` list. - -```python -# STORE YOUR EVALS - -# Tasks with subsets: -TASKS_TABLE = SUBSET_TASKS - -# Tasks without subsets: -# TASKS_TABLE = [task] -``` - -### Step 6: Creating a requirement file - -If your task has requirements, you need to create a `requirement.txt` file with -only the required dependencies so that anyone can run your task. - -## Running Your Custom Task - -Once your file is created, you can run the evaluation with the following command: - -```bash -lighteval accelerate \ - "model_name=HuggingFaceH4/zephyr-7b-beta" \ - {task} \ - --custom-tasks {path_to_your_custom_task_file} -``` - -### Example Usage - -```bash -# Run a custom task with 3 shot evaluation -lighteval accelerate \ - "model_name=openai-community/gpt2" \ - "myothertask|3" \ - --custom-tasks community_tasks/my_custom_task.py -``` diff --git a/docs/source/adding-a-new-metric.mdx b/docs/source/adding-a-new-metric.mdx deleted file mode 100644 index f81bf986b..000000000 --- a/docs/source/adding-a-new-metric.mdx +++ /dev/null @@ -1,163 +0,0 @@ -# Adding a New Metric - -## Before You Start - -### Two different types of metrics - -There are two types of metrics in Lighteval: - -#### Sample-Level Metrics -- **Purpose**: Evaluate individual samples/predictions -- **Input**: Takes a `Doc` and `ModelResponse` (model's prediction) -- **Output**: Returns a float or boolean value for that specific sample -- **Example**: Checking if a model's answer matches the correct answer for one sample - -#### Corpus-Level Metrics -- **Purpose**: Compute final scores across the entire dataset/corpus -- **Input**: Takes the results from all sample-level evaluations -- **Output**: Returns a single score representing overall performance -- **Examples**: - - Simple aggregation: Calculating average accuracy across all test samples - - Complex metrics: BLEU score where sample-level metric prepares data (tokenization, etc.) and corpus-level metric computes the actual BLEU score - -### Check Existing Metrics - -First, check if you can use one of the parameterized functions in -[Corpus Metrics](package_reference/metrics#corpus-metrics) or -[Sample Metrics](package_reference/metrics#sample-metrics). - -If not, you can use the `custom_task` system to register your new metric. - -> [!TIP] -> To see an example of a custom metric added along with a custom task, look at the [IFEval custom task](https://github.com/huggingface/lighteval/tree/main/examples/custom_tasks/ifeval). - -> [!WARNING] -> To contribute your custom metric to the Lighteval repository, you would first need -> to install the required dev dependencies by running `pip install -e .[dev]` -> and then run `pre-commit install` to install the pre-commit hooks. - -## Creating a Custom Metric - -### Step 1: Create the Metric File - -Create a new Python file which should contain the full logic of your metric. -The file also needs to start with these imports: - -```python -from aenum import extend_enum -from lighteval.metrics import Metrics -``` - -### Step 2: Define the Sample-Level Metric - -You need to define a sample-level metric. All sample-level metrics will have the same signature, taking a -[`~lighteval.types.Doc`] and a [`~lighteval.types.ModelResponse`]. The metric should return a float or a -boolean. - -#### Single Metric Example - -```python -def custom_metric(doc: Doc, model_response: ModelResponse) -> bool: - response = model_response.final_text[0] - return response == doc.choices[doc.gold_index] -``` - -#### Multiple Metrics Example - -If you want to return multiple metrics per sample, you need to return a dictionary with the metrics as keys and the values as values: - -```python -def custom_metric(doc: Doc, model_response: ModelResponse) -> dict: - response = model_response.final_text[0] - return {"accuracy": response == doc.choices[doc.gold_index], "other_metric": 0.5} -``` - -### Step 3: Define Aggregation Function (Optional) - -You can define an aggregation function if needed. A common aggregation function is `np.mean`: - -```python -def agg_function(items): - flat_items = [item for sublist in items for item in sublist] - score = sum(flat_items) / len(flat_items) - return score -``` - -### Step 4: Create the Metric Object - -#### Single Metric - -If it's a sample-level metric, you can use the following code -with [`~metrics.utils.metric_utils.SampleLevelMetric`]: - -```python -my_custom_metric = SampleLevelMetric( - metric_name="custom_accuracy", - higher_is_better=True, - category=SamplingMethod.GENERATIVE, - sample_level_fn=custom_metric, - corpus_level_fn=agg_function, -) -``` - -#### Multiple Metrics - -If your metric defines multiple metrics per sample, you can use the following code -with [`~metrics.utils.metric_utils.SampleLevelMetricGrouping`]: - -```python -custom_metric = SampleLevelMetricGrouping( - metric_name=["accuracy", "response_length", "confidence"], - higher_is_better={ - "accuracy": True, - "response_length": False, # Shorter responses might be better - "confidence": True - }, - category=SamplingMethod.GENERATIVE, - sample_level_fn=custom_metric, - corpus_level_fn={ - "accuracy": np.mean, - "response_length": np.mean, - "confidence": np.mean, - }, -) -``` - -### Step 5: Register the Metric - -To finish, add the following code so that it adds your metric to our metrics list -when loaded as a module: - -```python -# Adds the metric to the metric list! -extend_enum(Metrics, "CUSTOM_ACCURACY", my_custom_metric) - -if __name__ == "__main__": - print("Imported metric") -``` - -## Using Your Custom Metric - -### With Custom Tasks - -You can then give your custom metric to Lighteval by using `--custom-tasks -path_to_your_file` when launching it after adding it to the task config. - -```bash -lighteval accelerate \ - "model_name=openai-community/gpt2" \ - "truthfulqa:mc" \ - --custom-tasks path_to_your_metric_file.py -``` - -```python -from lighteval.tasks.lighteval_task import LightevalTaskConfig - -task = LightevalTaskConfig( - name="my_custom_task", - metric=[my_custom_metric], # Use your custom metric here - prompt_function=my_prompt_function, - hf_repo="my_dataset", - evaluation_splits=["test"] -) -``` diff --git a/docs/source/available-tasks.mdx b/docs/source/available-tasks.mdx deleted file mode 100644 index 8f5ce65f0..000000000 --- a/docs/source/available-tasks.mdx +++ /dev/null @@ -1,42 +0,0 @@ -# Available tasks - -Browse and inspect tasks available in LightEval. - - - - -List all tasks: - -```bash -lighteval tasks list -``` - -Extract tasks details: - -```bash -lighteval tasks dump -``` - -Store the tasks details in a JSON file: - -```bash -lighteval tasks dump > tasks.json -``` - -### Inspect specific tasks - -Inspect a task to view its config, metrics, and requirements: - -```bash -lighteval tasks inspect -``` - -Example: -```bash -lighteval tasks inspect truthfulqa:mc -``` diff --git a/docs/source/caching.mdx b/docs/source/caching.mdx deleted file mode 100644 index 9e5cbdb8c..000000000 --- a/docs/source/caching.mdx +++ /dev/null @@ -1,60 +0,0 @@ -# Caching System - -Lighteval includes a caching system that can significantly speed up evaluations by storing and reusing model predictions. -This is especially useful when running the same evaluation multiple times, or comparing different evaluation metrics on the same model outputs. - -## How It Works - -The caching system caches the predictions of the model for now (we will add tokenized input caching later). -It stores model responses objects (generations, logits, probabilities) for evaluation samples. - -### Cache Structure - -Cached data is stored on disk using HuggingFace datasets in the following structure: - -``` -.cache/ -└── huggingface/ - └── lighteval/ - └── predictions/ - └── {model_name}/ - └── {model_hash}/ - └── {task_name}.parquet -``` - -Where: -- `model_name`: The model name (path on the hub or local path) -- `model_hash`: Hash of the model configuration to ensure cache invalidation when parameters change -- `task_name`: Name of the evaluation task - -### Cache Recreation - -A new cache is automatically created when: -- Model configuration changes (different parameters, quantization, etc.) -- Model weights change (different revision, checkpoint, etc.) -- Generation parameters change (temperature, max_tokens, etc.) - -This ensures that cached results are always consistent with your current model setup. - -## Using Caching - -### Automatic Caching - -All built-in model classes in Lighteval automatically support caching. No additional configuration is needed. -For custom models you need to add a cache to the model class and decorators on all functions. - -## Cache Management - -### Clearing Cache - -To clear cache for a specific model, delete the corresponding directory: - -```bash -rm -rf ~/.cache/huggingface/lighteval/predictions/{model_name}/{model_hash}/ -``` - -To clear all caches: - -```bash -rm -rf ~/.cache/huggingface/lighteval/predictions -``` diff --git a/docs/source/contributing-to-multilingual-evaluations.mdx b/docs/source/contributing-to-multilingual-evaluations.mdx deleted file mode 100644 index 5490e846b..000000000 --- a/docs/source/contributing-to-multilingual-evaluations.mdx +++ /dev/null @@ -1,209 +0,0 @@ -# Contributing to Multilingual Evaluations - -Lighteval supports multilingual evaluations through a comprehensive system of translation literals and language-adapted templates. - -## Contributing Translation Literals - -### What Are Translation Literals? - -We define 19 `literals`, basic keywords or punctuation signs used when creating evaluation prompts in an automatic manner, such as `yes`, `no`, `because`, etc. - -These literals are essential for: -- **Consistent prompt formatting** across languages -- **Automatic prompt generation** for multilingual tasks -- **Proper localization** of evaluation templates - -### How to Contribute Translations - -We welcome translations in your language! To contribute: - -1. **Open the translation literals file**: [translation_literals.py](https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/templates/utils/translation_literals.py) - -2. **Edit the file** to add or expand the literal for your language of interest - -3. **Open a PR** with your modifications - -### Translation Literals Structure - -```python -Language.ENGLISH: TranslationLiterals( - language=Language.ENGLISH, - question_word="question", # Usage: "Question: How are you?" - answer="answer", # Usage: "Answer: I am fine" - confirmation_word="right", # Usage: "He is smart, right?" - yes="yes", # Usage: "Yes, he is" - no="no", # Usage: "No, he is not" - also="also", # Usage: "Also, she is smart." - cause_word="because", # Usage: "She is smart, because she is tall" - effect_word="therefore", # Usage: "He is tall therefore he is smart" - or_word="or", # Usage: "He is tall or small" - true="true", # Usage: "He is smart, true, false or neither?" - false="false", # Usage: "He is smart, true, false or neither?" - neither="neither", # Usage: "He is smart, true, false or neither?" - # Punctuation and spacing: only adjust if your language uses something different than in English - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - # The first characters of your alphabet used in enumerations, if different from English - indices=["A", "B", "C", ...] -) -``` - -## Contributing New Multilingual Tasks - -### Prerequisites - -Before creating a new multilingual task, you should: - -1. **Read the custom task guide**: [Adding a Custom Task](adding-a-custom-task) -2. **Understand multilingual task structure**: Review the [multilingual tasks](https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/multilingual/tasks.py) file -3. **Browse available templates**: Check the [templates directory](https://github.com/huggingface/lighteval/tree/main/src/lighteval/tasks/templates) - -### Key Concepts - -#### Language-Adapted Templates -For multilingual evaluations, the `prompt_function` should be implemented using language-adapted templates. These templates handle: -- **Correct formatting** for each language -- **Consistent usage** of language-adjusted prompt anchors (e.g., Question/Answer) -- **Proper punctuation** and spacing conventions - -#### Template Types -Available template types include: -- **XNLI**: Natural language inference tasks - [`get_nli_prompt_function`](https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/templates/nli.py#L162) -- **COPA**: Causal reasoning tasks - [`get_copa_prompt_function`](https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/templates/copa.py#L76) -- **Multiple Choice**: Standard multiple choice questions - [`get_mcq_prompt_function`](https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/templates/multichoice.py#L81) -- **Question Answering**: Open-ended question answering - [`get_qa_prompt_function`](https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/templates/qa.py#L46) -- **Custom**: Specialized task templates - -#### Formulation Types - -##### Multiple Choice Formulation (MCF) -Used for standard multiple choice questions where the model selects from lettered options: -```python -MCFFormulation() -``` - -**Example output:** -``` -Question: What is the capital of France? -A. London -B. Paris -C. Berlin -D. Rome -Answer: | A/B/C/D -``` - -##### Classification Formulation (CF) -Used for classification tasks where the model generates the answer directly: -```python -CFFormulation() -``` - -**Example output:** -``` -Question: What is the capital of France? -Answer: | Paris -``` - -##### Hybrid Formulation -Used for tasks that present choices but expect the full answer text: -```python -HybridFormulation() -``` - -**Example output:** -``` -Question: What is the capital of France? -A. London -B. Paris -C. Berlin -D. Rome -Answer: | Paris -``` - - -### Creating Your Multilingual Task - -#### Step 1: Create the Task File -Create a Python file following the custom task guide structure. - -#### Step 2: Import Required Components -```python -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.language import Language -from lighteval.tasks.multilingual.formulations import MCFFormulation, CFFormulation, HybridFormulation -from lighteval.tasks.multilingual.templates import get_template_prompt_function -from lighteval.tasks.multilingual.metrics import get_metrics_for_formulation, loglikelihood_acc_metric -from lighteval.tasks.multilingual.normalization import LogProbTokenNorm, LogProbCharNorm -``` - -#### Step 3: Define Your Tasks -```python -your_tasks = [ - LightevalTaskConfig( - # Name of your evaluation - name=f"evalname_{language.value}_{formulation.name.lower()}", - # This will automatically get the correct metrics for your chosen formulation - metric=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=None), - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - # In this function, you choose which template to follow and for which language and formulation - prompt_function=get_template_prompt_function( - language=language, - # Use the adapter to define the mapping between the - # keys of the template (left), and the keys of your dataset - # (right) - # To know which template keys are required and available, - # consult the appropriate adapter type and doc-string. - adapter=lambda line: { - "key": line["relevant_key"], - # Add more mappings as needed - }, - formulation=formulation, - ), - # You can also add specific filters to remove irrelevant samples - hf_filter=lambda line: line["label"] in , - # You then select your huggingface dataset as well as - # the splits available for evaluation - hf_repo=, - hf_subset=, - evaluation_splits=["train"], - hf_avail_splits=["train"], - ) - for language in [ - Language.YOUR_LANGUAGE, # Add your target languages - # Language.SPANISH, - # Language.FRENCH, - # etc. - ] - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] -] -``` - -#### Step 4: Test Your Implementation -Follow the custom task guide to test if your task is correctly implemented. - -> [!TIP] -> All [`~tasks.lighteval_task.LightevalTaskConfig`] parameters are strongly typed, including the inputs to the template function. Make sure to take advantage of your IDE's functionality to make it easier to correctly fill these parameters. - -### Validation Checklist -- [ ] Translation literals are accurate and complete -- [ ] Task works correctly across all target languages -- [ ] Metrics are appropriate for the task type -- [ ] Documentation is clear and comprehensive -- [ ] Code follows project conventions - -### Getting Help - -- **GitHub Issues**: Report bugs or ask questions -- **Discussions**: Join community discussions -- **Documentation**: Review existing guides and examples diff --git a/docs/source/evaluating-a-custom-model.mdx b/docs/source/evaluating-a-custom-model.mdx deleted file mode 100644 index df9b3ea17..000000000 --- a/docs/source/evaluating-a-custom-model.mdx +++ /dev/null @@ -1,207 +0,0 @@ -# Evaluating Custom Models - -Lighteval allows you to evaluate custom model implementations by creating a custom model class that inherits from `LightevalModel`. -This is useful when you want to evaluate models that aren't directly supported by the standard backends and providers (Transformers, VLLM, etc.), or -if you want to add your own pre/post-processing logic. - -## Creating a Custom Model - -### Step 1: Create Your Model Implementation - -Create a Python file containing your custom model implementation. The model must inherit from `LightevalModel` and implement all required methods. - -Here's a basic example: - -```python -from lighteval.models.abstract_model import LightevalModel -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.requests import Doc, SamplingMethod -from lighteval.utils.cache_management import SampleCache, cached - -class MyCustomModel(LightevalModel): - def __init__(self, config): - super().__init__(config) - # Initialize your model here... - - # Enable caching (recommended) - self._cache = SampleCache(config) - - @cached(SamplingMethod.GENERATIVE) - def greedy_until(self, docs: List[Doc]) -> List[ModelResponse]: - # Implement generation logic - pass - - @cached(SamplingMethod.LOGPROBS) - def loglikelihood(self, docs: List[Doc]) -> List[ModelResponse]: - # Implement loglikelihood computation - pass - - @cached(SamplingMethod.PERPLEXITY) - def loglikelihood_rolling(self, docs: List[Doc]) -> List[ModelResponse]: - # Implement rolling loglikelihood computation - pass -``` - -### Step 2: Model File Requirements - -The custom model file should contain exactly one class that inherits from `LightevalModel`. This class will be automatically detected and instantiated when loading the model. - -> [!TIP] -> You can find a complete example of a custom model implementation in `examples/custom_models/google_translate_model.py`. - -## Running the Evaluation - -You can evaluate your custom model using either the command-line interface or the Python API. - -### Using the Command Line - -```bash -lighteval custom \ - "google-translate" \ - "examples/custom_models/google_translate_model.py" \ - "wmt20:fr-de \ - --max-samples 10 -``` - -The command takes three required arguments: -- **Model name**: Used for tracking in results/logs -- **Model implementation file path**: Path to your Python file containing the custom model -- **Tasks**: Tasks to evaluate on (same format as other backends) - -### Using the Python API - -```python -from lighteval.logging.evaluation_tracker import EvaluationTracker -from lighteval.models.custom.custom_model import CustomModelConfig -from lighteval.pipeline import Pipeline, PipelineParameters, ParallelismManager - -# Set up evaluation tracking -evaluation_tracker = EvaluationTracker( - output_dir="results", - save_details=True -) - -# Configure the pipeline -pipeline_params = PipelineParameters( - launcher_type=ParallelismManager.CUSTOM, -) - -# Configure your custom model -model_config = CustomModelConfig( - model_name="my-custom-model", - model_definition_file_path="path/to/my_model.py" -) - -# Create and run the pipeline -pipeline = Pipeline( - tasks=truthfulqa:mc, - pipeline_parameters=pipeline_params, - evaluation_tracker=evaluation_tracker, - model_config=model_config -) - -pipeline.evaluate() -pipeline.save_and_push_results() -``` - -## Required Methods - -Your custom model must implement these core methods: - -### `greedy_until` -For generating text until a stop sequence or max tokens is reached. This is used for generative evaluations. - -```python -def greedy_until(self, docs: list[Doc]) -> list[ModelResponse]: - """ - Generate text until stop sequence or max tokens. - - Args: - docs: list of documents containing prompts and generation parameters - - Returns: - list of model responses with generated text - """ - pass -``` - -### `loglikelihood` -For computing log probabilities of specific continuations. This is used for multiple choice logprob evaluations. - -```python -def loglikelihood(self, docs: list[Doc]) -> list[ModelResponse]: - """ - Compute log probabilities of continuations. - - Args: - docs: list of documents containing context and continuation pairs - - Returns: - list of model responses with log probabilities - """ - pass -``` - -### `loglikelihood_rolling` -For computing rolling log probabilities of sequences. This is used for perplexity metrics. - -```python -def loglikelihood_rolling(self, docs: list[Doc]) -> list[ModelResponse]: - """ - Compute rolling log probabilities of sequences. - - Args: - docs: list of documents containing text sequences - - Returns: - list of model responses with rolling log probabilities - """ - pass -``` - -See the `LightevalModel` base class documentation for detailed method signatures and requirements. - -## Enabling Caching (Recommended) - -Lighteval includes a caching system that can significantly speed up evaluations by storing and reusing model predictions. -To enable caching in your custom model: - -### Step 1: Import Caching Components -```python -from lighteval.utils.cache_management import SampleCache, cached -``` - -### Step 2: Initialize Cache in Constructor -```python -def __init__(self, config): - super().__init__(config) - # Your initialization code... - self._cache = SampleCache(config) -``` - -3. Add cache decorators to your prediction methods: - ```python - @cached(SamplingMethod.GENERATIVE) - def greedy_until(self, docs: List[Doc]) -> List[ModelResponse]: - # Your implementation... - ``` - -For detailed information about the caching system, see the [Caching Documentation](caching). - -## Troubleshooting - -### Common Issues - -1. **Import Errors**: Ensure all required dependencies are installed -2. **Method Signature Errors**: Verify your methods match the expected signatures -3. **Caching Issues**: Check that cache decorators are applied correctly -4. **Performance Issues**: Consider implementing batching and caching - -### Debugging Tips - -- Use the `--max-samples` flag to test with a small dataset -- Enable detailed logging to see what's happening -- Test individual methods in isolation -- Check the example implementations for reference - -For more detailed information about custom model implementation, see the [Model Reference](package_reference/models). diff --git a/docs/source/index.mdx b/docs/source/index.mdx deleted file mode 100644 index 9eef93f5b..000000000 --- a/docs/source/index.mdx +++ /dev/null @@ -1,77 +0,0 @@ -# Lighteval - -🤗 Lighteval is your all-in-one toolkit for evaluating Large Language Models -(LLMs) across multiple backends with ease. Dive deep into your model's -performance by saving and exploring detailed, sample-by-sample results to debug -and see how your models stack up. - -> [!TIP] -> Share your evaluation results with the community by pushing them to the Hugging Face Hub. If you open Pull Requests on model repositories with evaluation results, we will automatically show the results on benchmark dataset repositories. Let's decentralize evaluation! Check out the [docs](https://huggingface.co/docs/hub/eval-results). - -## Key Features - -### 🚀 **Multi-Backend Support** -Evaluate your models using the most popular and efficient inference backends: -- `eval`: Use [inspect-ai](https://inspect.aisi.org.uk/) as backend to evaluate and inspect your models! (prefered way) -- `transformers`: Evaluate models on CPU or one or more GPUs using [🤗 - Accelerate](https://github.com/huggingface/transformers) -- `nanotron`: Evaluate models in distributed settings using [⚡️ - Nanotron](https://github.com/huggingface/nanotron) -- `vllm`: Evaluate models on one or more GPUs using [🚀 - VLLM](https://github.com/vllm-project/vllm) -- `custom`: Evaluate custom models (can be anything) -- `sglang`: Evaluate models using [SGLang](https://github.com/sgl-project/sglang) as backend -- `inference-endpoint`: Evaluate models using Hugging Face's [Inference Endpoints API](https://huggingface.co/inference-endpoints/dedicated) -- `tgi`: Evaluate models using [🔗 Text Generation Inference](https://huggingface.co/docs/text-generation-inference/en/index) running locally -- `litellm`: Evaluate models on any compatible API using [LiteLLM](https://www.litellm.ai/) -- `inference-providers`: Evaluate models using [HuggingFace's inference providers](https://huggingface.co/docs/inference-providers/en/index) as backend**: Distributed training and evaluation - -### 📊 **Comprehensive Evaluation** -- **Extensive Task Library**: 1000s pre-built evaluation tasks -- **Custom Task Creation**: Build your own evaluation tasks -- **Flexible Metrics**: Support for custom metrics and scoring -- **Detailed Analysis**: Sample-by-sample results for deep insights - -### 🔧 **Easy Customization** -Customization at your fingertips: create [new tasks](adding-a-custom-task), -[metrics](adding-a-new-metric) or [model](evaluating-a-custom-model) tailored to your needs, or browse all our existing tasks and metrics. - -### ☁️ **Seamless Integration** -Seamlessly experiment, benchmark, and store your results on the Hugging Face Hub, S3, or locally. - -## Quick Start - -### Installation - -```bash -pip install lighteval -``` - -### Basic Usage - -#### Find a task - - - -#### Run your benchmark and push details to the hub - -```bash -lighteval eval "hf-inference-providers/openai/gpt-oss-20b" \ - gpqa:diamond \ - --bundle-dir gpt-oss-bundle \ - --repo-id OpenEvals/evals -``` - -Resulting Space: - - diff --git a/docs/source/inspect-ai.mdx b/docs/source/inspect-ai.mdx deleted file mode 100644 index ff413dd84..000000000 --- a/docs/source/inspect-ai.mdx +++ /dev/null @@ -1,127 +0,0 @@ -# Evaluate your model with Inspect-AI - -Pick the right benchmarks with our benchmark finder: -Search by language, task type, dataset name, or keywords. - -> [!WARNING] -> Not all tasks are compatible with inspect-ai's API as of yet, we are working on converting all of them ! - - - - -Once you've chosen a benchmark, run it with `lighteval eval`. Below are examples for common setups. - -### Examples - -1. Evaluate a model via Hugging Face Inference Providers. - -```bash -lighteval eval "hf-inference-providers/openai/gpt-oss-20b" gpqa:diamond -``` - -2. Run multiple evals at the same time. - -```bash -lighteval eval "hf-inference-providers/openai/gpt-oss-20b" gpqa:diamond,aime25 -``` - -3. Compare providers for the same model. - -```bash -lighteval eval \ - hf-inference-providers/openai/gpt-oss-20b:fireworks-ai \ - hf-inference-providers/openai/gpt-oss-20b:together \ - hf-inference-providers/openai/gpt-oss-20b:nebius \ - gpqa:diamond -``` - -You can also compare every providers serving one model in one line: - -```bash - hf-inference-providers/openai/gpt-oss-20b:all \ - "lighteval|gpqa:diamond|0" -``` - -4. Evaluate a vLLM or SGLang model. - -```bash -lighteval eval vllm/HuggingFaceTB/SmolLM-135M-Instruct gpqa:diamond -``` - -5. See the impact of few-shot on your model. - -```bash -lighteval eval hf-inference-providers/openai/gpt-oss-20b "gsm8k|0,gsm8k|5" -``` - -6. Optimize custom server connections. - -```bash -lighteval eval hf-inference-providers/openai/gpt-oss-20b gsm8k \ - --max-connections 50 \ - --timeout 30 \ - --retry-on-error 1 \ - --max-retries 1 \ - --max-samples 10 -``` - -7. Use multiple epochs for more reliable results. - -```bash -lighteval eval hf-inference-providers/openai/gpt-oss-20b aime25 --epochs 16 --epochs-reducer "pass_at_4" -``` - -8. Push to the Hub to share results. - -```bash -lighteval eval hf-inference-providers/openai/gpt-oss-20b hle \ - --bundle-dir gpt-oss-bundle \ - --repo-id OpenEvals/evals \ - --max-samples 100 -``` - -Resulting Space: - - - -9. Change model behaviour - -You can use any argument defined in inspect-ai's API. - -```bash -lighteval eval hf-inference-providers/openai/gpt-oss-20b aime25 --temperature 0.1 -``` - -10. Use model-args to use any inference provider specific argument. - -```bash -lighteval eval google/gemini-2.5-pro aime25 --model-args location=us-east5 -``` - -```bash -lighteval eval openai/gpt-4o gpqa:diamond --model-args service_tier=flex,client_timeout=1200 -``` - - -LightEval prints a per-model results table: - -``` -Completed all tasks in 'lighteval-logs' successfully - -| Model |gpqa|gpqa:diamond| -|---------------------------------------|---:|-----------:| -|vllm/HuggingFaceTB/SmolLM-135M-Instruct|0.01| 0.01| - -results saved to lighteval-logs -run "inspect view --log-dir lighteval-logs" to view the results -``` diff --git a/docs/source/installation.mdx b/docs/source/installation.mdx deleted file mode 100644 index 0025c04f6..000000000 --- a/docs/source/installation.mdx +++ /dev/null @@ -1,71 +0,0 @@ -# Installation - -Lighteval can be installed from PyPI or from source. This guide covers all installation options and dependencies. - -## System Requirements - -- **Python**: 3.10 or higher -- **PyTorch**: 2.0 or higher (but less than 3.0) -- **CUDA**: Optional, for GPU acceleration - -## From PyPI - -The simplest way to install Lighteval is from PyPI: - -```bash -pip install lighteval -``` - -This installs the core package with all essential dependencies for basic evaluation tasks. - -## From Source - -Source installation is recommended for developers who want to contribute to Lighteval or need the latest features: - -```bash -git clone https://github.com/huggingface/lighteval.git -cd lighteval -pip install -e . -``` - -## Optional Dependencies (Extras) - -Lighteval provides several optional dependency groups that you can install based on your needs. Use the format `pip install lighteval[]` or `pip install -e .[]` for source installation. - -### Backend Extras - -| Extra | Description | Dependencies | -|-------|-------------|--------------| -| `vllm` | Use VLLM as backend for high-performance inference | vllm>=0.10.0, ray, more_itertools | -| `tgi` | Use Text Generation Inference API | text-generation>=0.6.0 | -| `litellm` | Use LiteLLM for unified API access | litellm, diskcache | -| `optimum` | Use Optimum for optimized models | optimum==1.12.0 | -| `quantization` | Evaluate quantized models | bitsandbytes>=0.41.0, auto-gptq>=0.4.2 | -| `adapters` | Evaluate adapter models (PEFT, Delta) | peft==0.3.0 | -| `nanotron` | Evaluate Nanotron models | nanotron, tensorboardX | - -### Task and Feature Extras - -| Extra | Description | Dependencies | -|-------|-------------|--------------| -| `extended_tasks` | Extended evaluation tasks | langdetect, openai>1.87, tiktoken | -| `multilingual` | Multilingual evaluation support | stanza, spacy[ja,ko,th], jieba, pyvi | -| `math` | Mathematical reasoning tasks | latex2sympy2_extended==1.0.6 | - -### Storage and Logging Extras - -| Extra | Description | Dependencies | -|-------|-------------|--------------| -| `s3` | Upload results to S3 | s3fs | -| `tensorboardX` | Upload results to TensorBoard | tensorboardX | -| `wandb` | Log results to Weights & Biases | wandb | -| `trackio` | Log results to Trackio | trackio | - -### Development Extras - -| Extra | Description | Dependencies | -|-------|-------------|--------------| -| `quality` | Code quality tools | ruff>=v0.11.0, pre-commit | -| `tests` | Testing dependencies | pytest>=7.4.0, deepdiff | -| `docs` | Documentation building | hf-doc-builder, watchdog | -| `dev` | All development dependencies | Includes accelerate, quality, tests, multilingual, math, extended_tasks, vllm | diff --git a/docs/source/metric-list.mdx b/docs/source/metric-list.mdx deleted file mode 100644 index 06d3dd069..000000000 --- a/docs/source/metric-list.mdx +++ /dev/null @@ -1,63 +0,0 @@ -# Metric List - -## Automatic metrics for multiple-choice tasks - -These metrics use log-likelihood of the different possible targets. -- `loglikelihood_acc`: Fraction of instances where the choice with the best logprob was correct - we recommend using a normalization by length -- `loglikelihood_f1`: Corpus level F1 score of the multichoice selection -- `mcc`: Matthew's correlation coefficient (a measure of agreement between statistical distributions). -- `recall_at_k`: Fraction of instances where the choice with the k-st best logprob or better was correct -- `mrr`: Mean reciprocal rank, a measure of the quality of a ranking of choices ordered by correctness/relevance -- `target_perplexity`: Perplexity of the different choices available. -- `acc_golds_likelihood`: A bit different, it actually checks if the average logprob of a single target is above or below 0.5. -- `multi_f1_numeric`: Loglikelihood F1 score for multiple gold targets. - -## Automatic metrics for perplexity and language modeling -These metrics use log-likelihood of prompt. -- `word_perplexity`: Perplexity (log probability of the input) weighted by the number of words of the sequence. -- `byte_perplexity`: Perplexity (log probability of the input) weighted by the number of bytes of the sequence. -- `bits_per_byte`: Average number of bits per byte according to model probabilities. -- `log_prob`: Predicted output's average log probability (input's log prob for language modeling). - -## Automatic metrics for generative tasks -These metrics need the model to generate an output. They are therefore slower. -- Base: - - `exact_match`: Fraction of instances where the prediction matches the gold. Several variations can be made through parametrization: - - normalization on string pre-comparision on whitespace, articles, capitalization, .... - - comparing the full string, or only subsets (prefix, suffix, ...) - - `maj_at_k`: Model majority vote. Samples k generations from the model and assumes the most frequent is the actual prediction. - - `f1_score`: Average F1 score in terms of word overlap between the model output and gold (normalisation optional). - - `f1_score_macro`: Corpus level macro F1 score. - - `f1_score_macro`: Corpus level micro F1 score. -- Summarization: - - `rouge`: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/). - - `rouge1`: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on 1-gram overlap. - - `rouge2`: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on 2-gram overlap. - - `rougeL`: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on longest common subsequence overlap. - - `rougeLsum`: Average ROUGE score [(Lin, 2004)](https://aclanthology.org/W04-1013/) based on longest common subsequence overlap. - - `rouge_t5` (BigBench): Corpus level ROUGE score for all available ROUGE metrics. - - `faithfulness`: Faithfulness scores based on the SummaC method of [Laban et al. (2022)](https://aclanthology.org/2022.tacl-1.10/). - - `extractiveness`: Reports, based on [(Grusky et al., 2018)](https://aclanthology.org/N18-1065/): - - `summarization_coverage`: Extent to which the model-generated summaries are extractive fragments from the source document, - - `summarization_density`: Extent to which the model-generated summaries are extractive summaries based on the source document, - - `summarization_compression`: Extent to which the model-generated summaries are compressed relative to the source document. - - `bert_score`: Reports the average BERTScore precision, recall, and f1 score [(Zhang et al., 2020)](https://openreview.net/pdf?id=SkeHuCVFDr) between model generation and gold summary. -- Translation: - - `bleu`: Corpus level BLEU score [(Papineni et al., 2002)](https://aclanthology.org/P02-1040/) - uses the sacrebleu implementation. - - `bleu_1`: Average sample BLEU score [(Papineni et al., 2002)](https://aclanthology.org/P02-1040/) based on 1-gram overlap - uses the nltk implementation. - - `bleu_4`: Average sample BLEU score [(Papineni et al., 2002)](https://aclanthology.org/P02-1040/) based on 4-gram overlap - uses the nltk implementation. - - `chrf`: Character n-gram matches f-score. - - `ter`: Translation edit/error rate. -- Copyright: - - `copyright`: Reports: - - `longest_common_prefix_length`: Average length of longest common prefix between model generation and reference, - - `edit_distance`: Average Levenshtein edit distance between model generation and reference, - - `edit_similarity`: Average Levenshtein edit similarity (normalized by the length of longer sequence) between model generation and reference. -- Math: - - Both `exact_match` and `maj_at_k` can be used to evaluate mathematics tasks with math specific normalization to remove and filter latex. - -## LLM-as-Judge -- `llm_judge_gpt3p5`: Can be used for any generative task, the model will be scored by a GPT3.5 model using the OpenAI API. -- `llm_judge_llama_3_405b`: Can be used for any generative task, the model will be scored by a Llama 3.405B model using the HuggingFace API. -- `llm_judge_multi_turn_gpt3p5`: Can be used for any generative task, the model will be scored by a GPT3.5 model using the OpenAI API. It is used for multiturn tasks like mt-bench. -- `llm_judge_multi_turn_llama_3_405b`: Can be used for any generative task, the model will be scored by a Llama 3.405B model using the HuggingFace API. It is used for multiturn tasks like mt-bench. diff --git a/docs/source/offline-evaluation.md b/docs/source/offline-evaluation.md deleted file mode 100644 index 37c921b61..000000000 --- a/docs/source/offline-evaluation.md +++ /dev/null @@ -1,46 +0,0 @@ -# Offline evaluation using local data files - -If you are prototyping a task based on files that are not yet hosted on the -Hub, you can take advantage of the `hf_data_files` argument to point lighteval -at local JSON/CSV resources. This makes it easy to evaluate datasets that live -in your repo or that are generated on the fly. - -Internally, `hf_data_files` is passed directly to the `data_files` parameter of `datasets.load_dataset` ([docs]((https://huggingface.co/docs/datasets/en/package_reference/loading_methods#datasets.load_dataset))). - -See [adding a custom task](adding-a-custom-task) for more information on how to create a custom task. - -```python -from pathlib import Path - -from lighteval.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def local_prompt(line: dict, task_name: str) -> Doc: - return Doc( - task_name=task_name, - query=line["question"], - choices=line["choices"], - gold_index=line["answer"] - ) - - -local_data = Path(__file__).parent / "samples" / "faq.jsonl" - -local_task = LightevalTaskConfig( - name="faq_eval", - prompt_function=local_prompt, - hf_repo="json", # Built-in streaming loader for json/jsonl files - hf_subset="default", - hf_data_files=str(local_data), # Can also be a dict mapping split names to paths - evaluation_splits=["train"], - metrics=[Metrics.ACCURACY], -) -``` - -Once the config is registered in `TASKS_TABLE`, running the task with -`--custom-tasks path/to/your_file.py` will automatically load the local data -files. You can also pass a dictionary to `hf_data_files` (e.g. -`{"train": "train.jsonl", "validation": "val.jsonl"}`) to expose multiple -splits. diff --git a/docs/source/package_reference/doc.mdx b/docs/source/package_reference/doc.mdx deleted file mode 100644 index 0a666ec83..000000000 --- a/docs/source/package_reference/doc.mdx +++ /dev/null @@ -1,3 +0,0 @@ -# Doc - -[[autodoc]] tasks.requests.Doc diff --git a/docs/source/package_reference/evaluation_tracker.mdx b/docs/source/package_reference/evaluation_tracker.mdx deleted file mode 100644 index 87dc90b7a..000000000 --- a/docs/source/package_reference/evaluation_tracker.mdx +++ /dev/null @@ -1,3 +0,0 @@ -# EvaluationTracker - -[[autodoc]] logging.evaluation_tracker.EvaluationTracker diff --git a/docs/source/package_reference/logging.mdx b/docs/source/package_reference/logging.mdx deleted file mode 100644 index 9102755c1..000000000 --- a/docs/source/package_reference/logging.mdx +++ /dev/null @@ -1,15 +0,0 @@ -# Logging - -## EvaluationTracker -[[autodoc]] logging.evaluation_tracker.EvaluationTracker - -## GeneralConfigLogger -[[autodoc]] logging.info_loggers.GeneralConfigLogger -## DetailsLogger -[[autodoc]] logging.info_loggers.DetailsLogger -## MetricsLogger -[[autodoc]] logging.info_loggers.MetricsLogger -## VersionsLogger -[[autodoc]] logging.info_loggers.VersionsLogger -## TaskConfigLogger -[[autodoc]] logging.info_loggers.TaskConfigLogger diff --git a/docs/source/package_reference/metrics.mdx b/docs/source/package_reference/metrics.mdx deleted file mode 100644 index bb22975bf..000000000 --- a/docs/source/package_reference/metrics.mdx +++ /dev/null @@ -1,76 +0,0 @@ -# Metrics - -## Metrics -[//]: # (TODO: aenum.Enum raises error when generating docs: not supported by inspect.signature. See: https://github.com/ethanfurman/aenum/issues/44) -[//]: # (### Metrics) -[//]: # ([[autodoc]] metrics.metrics.Metrics) -### Metric -[[autodoc]] metrics.utils.metric_utils.Metric -### CorpusLevelMetric -[[autodoc]] metrics.utils.metric_utils.CorpusLevelMetric -### SampleLevelMetric -[[autodoc]] metrics.utils.metric_utils.SampleLevelMetric -### MetricGrouping -[[autodoc]] metrics.utils.metric_utils.MetricGrouping -### CorpusLevelMetricGrouping -[[autodoc]] metrics.utils.metric_utils.CorpusLevelMetricGrouping -### SampleLevelMetricGrouping -[[autodoc]] metrics.utils.metric_utils.SampleLevelMetricGrouping - -## Corpus Metrics -### CorpusLevelF1Score -[[autodoc]] metrics.metrics_corpus.CorpusLevelF1Score -### CorpusLevelPerplexityMetric -[[autodoc]] metrics.metrics_corpus.CorpusLevelPerplexityMetric -### CorpusLevelTranslationMetric -[[autodoc]] metrics.metrics_corpus.CorpusLevelTranslationMetric -### MatthewsCorrCoef -[[autodoc]] metrics.metrics_corpus.MatthewsCorrCoef - -## Sample Metrics -### ExactMatches -[[autodoc]] metrics.metrics_sample.ExactMatches -### F1_score -[[autodoc]] metrics.metrics_sample.F1_score -### LoglikelihoodAcc -[[autodoc]] metrics.metrics_sample.LoglikelihoodAcc -### NormalizedMultiChoiceProbability -[[autodoc]] metrics.metrics_sample.NormalizedMultiChoiceProbability -### Probability -[[autodoc]] metrics.metrics_sample.Probability -### Recall -[[autodoc]] metrics.metrics_sample.Recall -### MRR -[[autodoc]] metrics.metrics_sample.MRR -### ROUGE -[[autodoc]] metrics.metrics_sample.ROUGE -### BertScore -[[autodoc]] metrics.metrics_sample.BertScore -### Extractiveness -[[autodoc]] metrics.metrics_sample.Extractiveness -### Faithfulness -[[autodoc]] metrics.metrics_sample.Faithfulness -### BLEURT -[[autodoc]] metrics.metrics_sample.BLEURT -### BLEU -[[autodoc]] metrics.metrics_sample.BLEU -### StringDistance -[[autodoc]] metrics.metrics_sample.StringDistance - -### Metrics allowing sampling -#### PassAtK -[[autodoc]] metrics.metrics_sample.PassAtK -#### MajAtN -[[autodoc]] metrics.metrics_sample.MajAtN -#### AvgAtN -[[autodoc]] metrics.metrics_sample.AvgAtN - -## LLM-as-a-Judge -### JudgeLM -[[autodoc]] metrics.utils.llm_as_judge.JudgeLM -### JudgeLLM -[[autodoc]] metrics.metrics_sample.JudgeLLM -### JudgeLLMMTBench -[[autodoc]] metrics.metrics_sample.JudgeLLMMTBench -### JudgeLLMMixEval -[[autodoc]] metrics.metrics_sample.JudgeLLMMixEval diff --git a/docs/source/package_reference/models.mdx b/docs/source/package_reference/models.mdx deleted file mode 100644 index 7655396e5..000000000 --- a/docs/source/package_reference/models.mdx +++ /dev/null @@ -1,43 +0,0 @@ -# Model Configs - -The model configs are used to define the model and its parameters. All the parameters can be -set in the `model-args` or in the model yaml file (see example -[here](https://github.com/huggingface/lighteval/blob/main/examples/model_configs/vllm_model_config.yaml)). - -### Base model config -[[autodoc]] models.abstract_model.ModelConfig - -## Local Models - -### Transformers Model -[[autodoc]] models.transformers.transformers_model.TransformersModelConfig -[[autodoc]] models.transformers.adapter_model.AdapterModelConfig -[[autodoc]] models.transformers.delta_model.DeltaModelConfig - -### VLLM Model -[[autodoc]] models.vllm.vllm_model.VLLMModelConfig - -### SGLang Model -[[autodoc]] models.sglang.sglang_model.SGLangModelConfig - -### Dummy Model -[[autodoc]] models.dummy.dummy_model.DummyModelConfig - - -## Endpoints-based Models - -### Inference Providers Model -[[autodoc]] models.endpoints.inference_providers_model.InferenceProvidersModelConfig - -### InferenceEndpointModel -[[autodoc]] models.endpoints.endpoint_model.InferenceEndpointModelConfig -[[autodoc]] models.endpoints.endpoint_model.ServerlessEndpointModelConfig - -### TGI ModelClient -[[autodoc]] models.endpoints.tgi_model.TGIModelConfig - -### Litellm Model -[[autodoc]] models.endpoints.litellm_model.LiteLLMModelConfig - -## Custom Model -[[autodoc]] models.custom.custom_model.CustomModelConfig diff --git a/docs/source/package_reference/models_outputs.mdx b/docs/source/package_reference/models_outputs.mdx deleted file mode 100644 index 8bc0737a1..000000000 --- a/docs/source/package_reference/models_outputs.mdx +++ /dev/null @@ -1,5 +0,0 @@ -# Model's Output - -All models will generate an ouput per Doc supplied to the `generation` or `loglikelihood` fuctions. - -[[autodoc]] lighteval.models.model_output.ModelResponse diff --git a/docs/source/package_reference/pipeline.mdx b/docs/source/package_reference/pipeline.mdx deleted file mode 100644 index 3473abb3d..000000000 --- a/docs/source/package_reference/pipeline.mdx +++ /dev/null @@ -1,13 +0,0 @@ -# Pipeline - -## Pipeline - -[[autodoc]] pipeline.Pipeline - -## PipelineParameters - -[[autodoc]] pipeline.PipelineParameters - -## ParallelismManager - -[[autodoc]] pipeline.ParallelismManager diff --git a/docs/source/package_reference/tasks.mdx b/docs/source/package_reference/tasks.mdx deleted file mode 100644 index 96b082b3c..000000000 --- a/docs/source/package_reference/tasks.mdx +++ /dev/null @@ -1,23 +0,0 @@ -# Tasks - -## LightevalTask -### LightevalTaskConfig -[[autodoc]] tasks.lighteval_task.LightevalTaskConfig -### LightevalTask -[[autodoc]] tasks.lighteval_task.LightevalTask - -## PromptManager -[[autodoc]] tasks.prompt_manager.PromptManager - -## Registry -[[autodoc]] tasks.registry.Registry - -## Doc -[[autodoc]] tasks.requests.Doc - -## Datasets -[[autodoc]] data.DynamicBatchDataset -[[autodoc]] data.LoglikelihoodDataset -[[autodoc]] data.GenerativeTaskDataset -[[autodoc]] data.GenerativeTaskDatasetNanotron -[[autodoc]] data.GenDistributedSampler diff --git a/docs/source/quicktour.mdx b/docs/source/quicktour.mdx deleted file mode 100644 index 919c8a1a1..000000000 --- a/docs/source/quicktour.mdx +++ /dev/null @@ -1,138 +0,0 @@ -# Quick Tour - -> [!TIP] -> We recommend using the `--help` flag to get more information about the -> available options for each command. -> `lighteval --help` - -Lighteval can be used with several different commands, each optimized for different evaluation scenarios. - - -## Find your benchmark - - - -## Available Commands - -### Evaluation Backends - -- `lighteval eval`: Use [inspect-ai](https://inspect.aisi.org.uk/) as backend to evaluate and inspect your models ! (prefered way) -- `lighteval accelerate`: Evaluate models on CPU or one or more GPUs using [🤗 - Accelerate](https://github.com/huggingface/accelerate) -- `lighteval nanotron`: Evaluate models in distributed settings using [⚡️ - Nanotron](https://github.com/huggingface/nanotron) -- `lighteval vllm`: Evaluate models on one or more GPUs using [🚀 - VLLM](https://github.com/vllm-project/vllm) -- `lighteval custom`: Evaluate custom models (can be anything) -- `lighteval sglang`: Evaluate models using [SGLang](https://github.com/sgl-project/sglang) as backend -- `lighteval endpoint`: Evaluate models using various endpoints as backend - - `lighteval endpoint inference-endpoint`: Evaluate models using Hugging Face's [Inference Endpoints API](https://huggingface.co/inference-endpoints/dedicated) - - `lighteval endpoint tgi`: Evaluate models using [🔗 Text Generation Inference](https://huggingface.co/docs/text-generation-inference/en/index) running locally - - `lighteval endpoint litellm`: Evaluate models on any compatible API using [LiteLLM](https://www.litellm.ai/) - - `lighteval endpoint inference-providers`: Evaluate models using [HuggingFace's inference providers](https://huggingface.co/docs/inference-providers/en/index) as backend - -### Evaluation Utils - -- `lighteval baseline`: Compute baselines for given tasks - -### Utils - -- `lighteval tasks`: List or inspect tasks - - `lighteval tasks list`: List all available tasks - - `lighteval tasks inspect`: Inspect a specific task to see its configuration and samples - - `lighteval tasks create`: Create a new task from a template - -## Basic Usage - -To evaluate `GPT-2` on the Truthful QA benchmark with [🤗 - Accelerate](https://github.com/huggingface/accelerate), run: - -```bash -lighteval accelerate \ - "model_name=openai-community/gpt2" \ - truthfulqa:mc -``` - -### Task Specification - -Tasks have a function applied at the sample level and one at the corpus level. For example, -- an exact match can be applied per sample, then averaged over the corpus to give the final score -- samples can be left untouched before applying Corpus BLEU at the corpus level -etc. - -If the task you are looking at has a sample level function (`sample_level_fn`) which can be parametrized, you can pass parameters in the CLI. -For example -```txt -{task}@{parameter_name1}={value1}@{parameter_name2}={value2},...|0 -``` - -All officially supported tasks can be found at the [tasks_list](available-tasks) and in the -[extended folder](https://github.com/huggingface/lighteval/tree/main/src/lighteval/tasks/extended). -Moreover, community-provided tasks can be found in the -[community](https://github.com/huggingface/lighteval/tree/main/community_tasks) folder. - -For more details on the implementation of the tasks, such as how prompts are constructed or which metrics are used, you can examine the -[implementation file](https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/default_tasks.py). - -### Running Multiple Tasks - -Running multiple tasks is supported, either with a comma-separated list or by specifying a file path. -The file should be structured like [examples/tasks/recommended_set.txt](https://github.com/huggingface/lighteval/blob/main/examples/tasks/recommended_set.txt). -When specifying a path to a file, it should start with `./`. - -```bash -lighteval accelerate \ - "model_name=openai-community/gpt2" \ - ./path/to/lighteval/examples/tasks/recommended_set.txt -# or, e.g., "truthfulqa:mc|0,gsm8k|3" -``` - -## Backend Configuration - -### General Information - -The `model-args` argument takes a string representing a list of model -arguments. The arguments allowed vary depending on the backend you use and -correspond to the fields of the model configurations. - -The model configurations can be found [here](./package_reference/models). - -All models allow you to post-process your reasoning model predictions -to remove the thinking tokens from the trace used to compute the metrics, -using `--remove-reasoning-tags` and `--reasoning-tags` to specify which -reasoning tags to remove (defaults to `` and ``). - -Here's an example with `mistralai/Magistral-Small-2507` which outputs custom -thinking tokens: - -```bash -lighteval vllm \ - "model_name=mistralai/Magistral-Small-2507,dtype=float16,data_parallel_size=4" \ - aime24 \ - --remove-reasoning-tags \ - --reasoning-tags="[('[THINK]','[/THINK]')]" -``` - -### Nanotron - -To evaluate a model trained with Nanotron on a single GPU: - -> [!WARNING] -> Nanotron models cannot be evaluated without torchrun. - -```bash -torchrun --standalone --nnodes=1 --nproc-per-node=1 \ - src/lighteval/__main__.py nanotron \ - --checkpoint-config-path ../nanotron/checkpoints/10/config.yaml \ - --lighteval-config-path examples/nanotron/lighteval_config_override_template.yaml -``` - -The `nproc-per-node` argument should match the data, tensor, and pipeline -parallelism configured in the `lighteval_config_template.yaml` file. -That is: `nproc-per-node = data_parallelism * tensor_parallelism * -pipeline_parallelism`. diff --git a/docs/source/saving-and-reading-results.mdx b/docs/source/saving-and-reading-results.mdx deleted file mode 100644 index 167cf12e0..000000000 --- a/docs/source/saving-and-reading-results.mdx +++ /dev/null @@ -1,280 +0,0 @@ -# Saving and Reading Results - -Lighteval provides comprehensive logging and result management through the `EvaluationTracker` class. This system allows you to save results locally and optionally push them to various platforms for collaboration and analysis. - -## Saving Results Locally - -Lighteval automatically saves results and evaluation details in the -directory specified with the `--output-dir` option. The results are saved in -`{output_dir}/results/{model_name}/results_{timestamp}.json`. [Here is an -example of a result file](#example-of-a-result-file). The output path can be -any [fsspec](https://filesystem-spec.readthedocs.io/en/latest/index.html) -compliant path (local, S3, Hugging Face Hub, Google Drive, FTP, etc.). - -To save detailed evaluation information, you can use the `--save-details` -option. The details are saved in Parquet files at -`{output_dir}/details/{model_name}/{timestamp}/details_{task}_{timestamp}.parquet`. - -If you want results to be saved in a custom path structure, you can set the `results-path-template` option. -This allows you to specify a string template for the path. The template must contain the following -variables: `output_dir`, `model_name`, `org`. For example: -`{output_dir}/{org}_{model}`. The template will be used to create the path for the results file. - -## Pushing Results to the Hugging Face Hub - -You can push results and evaluation details to the Hugging Face Hub. To do -so, you need to set the `--push-to-hub` option as well as the `--results-org` -option. The results are saved in a dataset with the name -`{results_org}/{model_org}/{model_name}`. To push the details, you need to set -the `--save-details` option. - -The dataset created will be private by default. You can make it public by -setting the `--public-run` option. - -## Pushing Results to TensorBoard - -You can push results to TensorBoard by setting `--push-to-tensorboard`. -This creates a TensorBoard dashboard in a Hugging Face organization specified with the `--results-org` -option. - -## Pushing Results to Weights & Biases or Trackio - -You can push results to Weights & Biases by setting `--wandb`. This initializes a W&B -run and logs the results. - -W&B arguments need to be set in your environment variables: - -```bash -export WANDB_PROJECT="lighteval" -``` - -You can find a complete list of variables in the [W&B documentation](https://docs.wandb.ai/guides/track/environment-variables/). - -If Trackio is available in your environment (`pip install lighteval[trackio]`), it will be used to log and push results to a -Hugging Face dataset. Choose the dataset name and organization with: - -```bash -export WANDB_SPACE_ID="org/name" -``` - -## How to Load and Investigate Details - -### Loading from Local Detail Files - -```python -from datasets import load_dataset -import os -import glob - -output_dir = "evals_doc" -model_name = "HuggingFaceH4/zephyr-7b-beta" -timestamp = "latest" -task = "gsm8k" - -if timestamp == "latest": - path = f"{output_dir}/details/{model_name}/*/" - timestamps = glob.glob(path) - timestamp = sorted(timestamps)[-1].split("/")[-2] - print(f"Latest timestamp: {timestamp}") - -details_path = f"{output_dir}/details/{model_name}/{timestamp}/details_{task}_{timestamp}.parquet" - -# Load the details -details = load_dataset("parquet", data_files=details_path, split="train") - -for detail in details: - print(detail) -``` - -### Loading from the Hugging Face Hub - -```python -from datasets import load_dataset - -results_org = "SaylorTwift" -model_name = "HuggingFaceH4/zephyr-7b-beta" -sanitized_model_name = model_name.replace("/", "__") -task = "gsm8k" -public_run = False - -dataset_path = f"{results_org}/details_{sanitized_model_name}{'_private' if not public_run else ''}" -details = load_dataset(dataset_path, task.replace("|", "_"), split="latest") - -for detail in details: - print(detail) -``` - -## Detail File Structure - -The detail file contains the following columns: - -- **`__doc__`**: The document used for evaluation, containing the gold reference, few-shot examples, and other hyperparameters used for the task. -- **`__model_response__`**: Contains model generations, log probabilities, and the input that was sent to the model. -- **`__metric__`**: The value of the metrics for this sample. - -## EvaluationTracker Configuration - -The `EvaluationTracker` class provides several configuration options for customizing how results are saved and pushed: - -### Basic Configuration - -```python -from lighteval.logging.evaluation_tracker import EvaluationTracker - -tracker = EvaluationTracker( - output_dir="./results", - save_details=True, - push_to_hub=True, - hub_results_org="your_username", - public=False -) -``` - -### Advanced Configuration - -```python -tracker = EvaluationTracker( - output_dir="./results", - results_path_template="{output_dir}/custom/{org}_{model}", - save_details=True, - push_to_hub=True, - push_to_tensorboard=True, - hub_results_org="my-org", - tensorboard_metric_prefix="eval", - public=True, - use_wandb=True -) -``` - -### Key Parameters - -- **`output_dir`**: Local directory to save evaluation results and logs -- **`results_path_template`**: Template for results directory structure -- **`save_details`**: Whether to save detailed evaluation records (default: True) -- **`push_to_hub`**: Whether to push results to Hugging Face Hub (default: False) -- **`push_to_tensorboard`**: Whether to push metrics to TensorBoard (default: False) -- **`hub_results_org`**: Hugging Face Hub organization to push results to -- **`tensorboard_metric_prefix`**: Prefix for TensorBoard metrics (default: "eval") -- **`public`**: Whether to make Hub datasets public (default: False) -- **`use_wandb`**: Whether to log to Weights & Biases or Trackio (default: False) - -## Result File Structure - -The main results file contains several sections: - -### General Configuration -- **`config_general`**: Overall evaluation configuration including model information, timing, and system details -- **`summary_general`**: General statistics about the evaluation run - -### Task-Specific Information -- **`config_tasks`**: Configuration details for each evaluated task -- **`summary_tasks`**: Task-specific statistics and metadata -- **`versions`**: Version information for tasks and datasets - -### Results -- **`results`**: Actual evaluation metrics and scores for each task - -## Example of a Result File - -```json -{ - "config_general": { - "lighteval_sha": "203045a8431bc9b77245c9998e05fc54509ea07f", - "num_fewshot_seeds": 1, - "max_samples": 1, - "job_id": "", - "start_time": 620979.879320166, - "end_time": 621004.632108041, - "total_evaluation_time_secondes": "24.752787875011563", - "model_name": "gpt2", - "model_sha": "607a30d783dfa663caf39e06633721c8d4cfcd7e", - "model_dtype": null, - "model_size": "476.2 MB" - }, - "results": { - "gsm8k|0": { - "em": 0.0, - "em_stderr": 0.0, - "maj@8": 0.0, - "maj@8_stderr": 0.0 - }, - "all": { - "em": 0.0, - "em_stderr": 0.0, - "maj@8": 0.0, - "maj@8_stderr": 0.0 - } - }, - "versions": { - "gsm8k|0": 0 - }, - "config_tasks": { - "lighteval|gsm8k": { - "name": "gsm8k", - "prompt_function": "gsm8k", - "hf_repo": "gsm8k", - "hf_subset": "main", - "metric": [ - { - "metric_name": "em", - "higher_is_better": true, - "category": "3", - "use_case": "5", - "sample_level_fn": "compute", - "corpus_level_fn": "mean" - }, - { - "metric_name": "maj@8", - "higher_is_better": true, - "category": "5", - "use_case": "5", - "sample_level_fn": "compute", - "corpus_level_fn": "mean" - } - ], - "hf_avail_splits": [ - "train", - "test" - ], - "evaluation_splits": [ - "test" - ], - "few_shots_split": null, - "few_shots_select": "random_sampling_from_train", - "generation_size": 256, - "generation_grammar": null, - "stop_sequence": [ - "Question=" - ], - "num_samples": null, - "original_num_docs": 1319, - "effective_num_docs": 1, - "must_remove_duplicate_docs": null, - "version": 0 - } - }, - "summary_tasks": { - "gsm8k|0": { - "hashes": { - "hash_examples": "8517d5bf7e880086", - "hash_full_prompts": "8517d5bf7e880086", - "hash_input_tokens": "29916e7afe5cb51d", - "hash_cont_tokens": "37f91ce23ef6d435" - }, - "padded": 0, - "non_padded": 2, - "effective_few_shots": 0.0, - } - }, - "summary_general": { - "hashes": { - "hash_examples": "5f383c395f01096e", - "hash_full_prompts": "5f383c395f01096e", - "hash_input_tokens": "ac933feb14f96d7b", - "hash_cont_tokens": "9d03fb26f8da7277" - }, - "padded": 0, - "non_padded": 2, - } -} -``` diff --git a/docs/source/use-huggingface-inference-endpoints-or-tgi-as-backend.mdx b/docs/source/use-huggingface-inference-endpoints-or-tgi-as-backend.mdx deleted file mode 100644 index 7c1ab95f7..000000000 --- a/docs/source/use-huggingface-inference-endpoints-or-tgi-as-backend.mdx +++ /dev/null @@ -1,156 +0,0 @@ -# Using Hugging Face Inference Endpoints or TGI as Backend - -An alternative to launching the evaluation locally is to serve the model on a -TGI-compatible server/container and then run the evaluation by sending requests -to the server. The command is the same as before, except you specify a path to -a YAML configuration file (detailed below): - -```bash -lighteval endpoint {tgi,inference-endpoint} \ - "/path/to/config/file" \ - -``` - -There are two types of configuration files that can be provided for running on -the server: - -## Hugging Face Inference Endpoints - -To launch a model using Hugging Face's Inference Endpoints, you need to provide -the following file: `endpoint_model.yaml`. Lighteval will automatically deploy -the endpoint, run the evaluation, and finally delete the endpoint (unless you -specify an endpoint that was already launched, in which case the endpoint won't -be deleted afterwards). - -### Configuration File Example - -```yaml -model_parameters: - reuse_existing: false # If true, ignore all params in instance, and don't delete the endpoint after evaluation - # endpoint_name: "llama-2-7B-lighteval" # Needs to be lowercase without special characters - model_name: "meta-llama/Llama-2-7b-hf" - revision: "main" # Defaults to "main" - dtype: "float16" # Can be any of "awq", "eetq", "gptq", "4bit" or "8bit" (will use bitsandbytes), "bfloat16" or "float16" - accelerator: "gpu" - region: "eu-west-1" - vendor: "aws" - instance_type: "nvidia-a10g" - instance_size: "x1" - framework: "pytorch" - endpoint_type: "protected" - namespace: null # The namespace under which to launch the endpoint. Defaults to the current user's namespace - image_url: null # Optionally specify the docker image to use when launching the endpoint model. E.g., launching models with later releases of the TGI container with support for newer models. - env_vars: null # Optional environment variables to include when launching the endpoint. e.g., `MAX_INPUT_LENGTH: 2048` -``` - -## Text Generation Inference (TGI) - -To use a model already deployed on a TGI server, for example on Hugging Face's -serverless inference. - -### Configuration File Example - -```yaml -model_parameters: - inference_server_address: "" - inference_server_auth: null - model_id: null # Optional, only required if the TGI container was launched with model_id pointing to a local directory -``` - -## Key Parameters - -### Hugging Face Inference Endpoints - -#### Model Configuration -- `model_name`: The Hugging Face model ID to deploy -- `revision`: Model revision (defaults to "main") -- `dtype`: Data type for model weights ("float16", "bfloat16", "4bit", "8bit", etc.) -- `framework`: Framework to use ("pytorch", "tensorflow") - -#### Infrastructure Settings -- `accelerator`: Hardware accelerator ("gpu", "cpu") -- `region`: AWS region for deployment -- `vendor`: Cloud vendor ("aws", "azure", "gcp") -- `instance_type`: Instance type (e.g., "nvidia-a10g", "nvidia-t4") -- `instance_size`: Instance size ("x1", "x2", etc.) - -#### Endpoint Configuration -- `endpoint_type`: Endpoint access level ("public", "protected", "private") -- `namespace`: Organization namespace for deployment -- `reuse_existing`: Whether to reuse an existing endpoint -- `endpoint_name`: Custom endpoint name (lowercase, no special characters) - -#### Advanced Settings -- `image_url`: Custom Docker image URL -- `env_vars`: Environment variables for the endpoint - -### Text Generation Inference (TGI) - -#### Server Configuration -- `inference_server_address`: URL of the TGI server -- `inference_server_auth`: Authentication credentials -- `model_id`: Model identifier (if using local model directory) - -## Usage Examples - -### Deploying a New Inference Endpoint - -```bash -lighteval endpoint inference-endpoint \ - "configs/endpoint_model.yaml" \ - gsm8k -``` - -### Using an Existing TGI Server - -```bash -lighteval endpoint tgi \ - "configs/tgi_server.yaml" \ - gsm8k -``` - -### Reusing an Existing Endpoint - -```yaml -model_parameters: - reuse_existing: true - endpoint_name: "my-existing-endpoint" - # Other parameters will be ignored when reuse_existing is true -``` - -## Cost Management - -### Inference Endpoints -- Endpoints are automatically deleted after evaluation (unless `reuse_existing: true`) -- Costs are based on instance type and runtime -- Monitor usage in the [Hugging Face billing dashboard](https://huggingface.co/settings/billing) - -### TGI Servers -- No additional costs beyond your existing server infrastructure -- Useful for cost-effective evaluation of already-deployed models - -## Troubleshooting - -### Common Issues - -1. **Endpoint Deployment Failures**: Check instance availability in your region -2. **Authentication Errors**: Ensure proper Hugging Face token permissions -3. **Model Loading Errors**: Verify model name and revision are correct -4. **Resource Constraints**: Choose appropriate instance type for your model size - -### Performance Tips - -- Use appropriate instance types for your model size -- Consider using quantized models (4bit, 8bit) for cost savings -- Reuse existing endpoints for multiple evaluations -- Use serverless TGI for cost-effective evaluation - -### Error Handling - -Common error messages and solutions: -- **"Instance not available"**: Try a different region or instance type -- **"Model not found"**: Check the model name and revision -- **"Insufficient permissions"**: Verify your Hugging Face token has endpoint deployment permissions -- **"Endpoint already exists"**: Use `reuse_existing: true` or choose a different endpoint name - -For more detailed information about Hugging Face Inference Endpoints, see the [official documentation](https://huggingface.co/docs/inference-endpoints/). diff --git a/docs/source/use-inference-providers-as-backend.mdx b/docs/source/use-inference-providers-as-backend.mdx deleted file mode 100644 index 70a0abd6a..000000000 --- a/docs/source/use-inference-providers-as-backend.mdx +++ /dev/null @@ -1,69 +0,0 @@ -# Using Inference Providers as Backend - -Lighteval allows you to use Hugging Face's Inference Providers to evaluate LLMs on supported providers such as Black Forest Labs, Cerebras, Fireworks AI, Nebius, Together AI, and many more. - -> [!WARNING] -> Do not forget to set your Hugging Face API key. -> You can set it using the `HF_TOKEN` environment variable or by using the `huggingface-cli` command. - -## Basic Usage - -```bash -lighteval endpoint inference-providers \ - "model_name=deepseek-ai/DeepSeek-R1,provider=hf-inference" \ - gsm8k -``` - -## Using a Configuration File - -You can use configuration files to define the model and the provider to use. - -```bash -lighteval endpoint inference-providers \ - examples/model_configs/inference_providers.yaml \ - gsm8k -``` - -With the following configuration file: - -```yaml -model_parameters: - model_name: "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" - provider: "novita" - timeout: null - proxies: null - parallel_calls_count: 10 - generation_parameters: - temperature: 0.8 - top_k: 10 - max_new_tokens: 10000 -``` - -By default, inference requests are billed to your personal account. -Optionally, you can charge them to an organization by setting `org_to_bill=""` (requires being a member of that organization). - -## Supported Providers - -Hugging Face Inference Providers supports a wide range of LLM providers see the [Inference Providers documentation](https://huggingface.co/docs/inference-providers/en/index) for the complete list. - - -## Billing and Costs - -### Personal Account Billing -By default, all inference requests are billed to your personal Hugging Face account. You can monitor your usage in the [Hugging Face billing dashboard](https://huggingface.co/settings/billing). - -### Organization Billing -To bill requests to an organization: - -1. Ensure you are a member of the organization -2. Add `org_to_bill=""` to your configuration -3. The organization must have sufficient credits - -```yaml -model_parameters: - model_name: "meta-llama/Llama-2-7b-chat-hf" - provider: "together" - org_to_bill: "my-organization" -``` - -For more detailed error handling and provider-specific information, refer to the [Hugging Face Inference Providers documentation](https://huggingface.co/docs/inference-providers/en/index). diff --git a/docs/source/use-litellm-as-backend.mdx b/docs/source/use-litellm-as-backend.mdx deleted file mode 100644 index 257da5d32..000000000 --- a/docs/source/use-litellm-as-backend.mdx +++ /dev/null @@ -1,72 +0,0 @@ -# Using LiteLLM as Backend - -Lighteval allows you to use LiteLLM as a backend, enabling you to call all LLM APIs -using the OpenAI format. LiteLLM supports various providers including Bedrock, Hugging Face, Vertex AI, Together AI, Azure, -OpenAI, Groq, and many others. - -> [!TIP] -> Documentation for available APIs and compatible endpoints can be found [here](https://docs.litellm.ai/docs/). - -## Basic Usage - -```bash -lighteval endpoint litellm \ - "provider=openai,model_name=gpt-3.5-turbo" \ - gsm8k -``` - -## Using a Configuration File - -LiteLLM allows generation with any OpenAI-compatible endpoint. For example, you -can evaluate a model running on a local VLLM server. - -To do so, you will need to use a configuration file like this: - -```yaml -model_parameters: - model_name: "openai/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" - base_url: "URL_OF_THE_ENDPOINT_YOU_WANT_TO_USE" - api_key: "" # Remove or keep empty as needed - generation_parameters: - temperature: 0.5 - max_new_tokens: 256 - stop_tokens: [""] - top_p: 0.9 - seed: 0 - repetition_penalty: 1.0 - frequency_penalty: 0.0 -``` - -## Supported Providers - -LiteLLM supports a wide range of LLM providers: - -### Cloud Providers - -all cloud providers can be found in the [litellm documentation](https://docs.litellm.ai/docs/providers). - -### Local/On-Premise -- **VLLM**: Local VLLM servers -- **Hugging Face**: Local Hugging Face models -- **Custom endpoints**: Any OpenAI-compatible API - -## Using with Local Models - -### VLLM Server -To use with a local VLLM server: - -1. Start your VLLM server: -```bash -vllm serve HuggingFaceH4/zephyr-7b-beta --host 0.0.0.0 --port 8000 -``` - -2. Configure LiteLLM to use the local server: -```yaml -model_parameters: - provider: "hosted_vllm" - model_name: "hosted_vllm/HuggingFaceH4/zephyr-7b-beta" - base_url: "http://localhost:8000/v1" - api_key: "" -``` - -For more detailed error handling and debugging, refer to the [LiteLLM documentation](https://docs.litellm.ai/docs/). diff --git a/docs/source/use-sglang-as-backend.mdx b/docs/source/use-sglang-as-backend.mdx deleted file mode 100644 index 129b06b87..000000000 --- a/docs/source/use-sglang-as-backend.mdx +++ /dev/null @@ -1,112 +0,0 @@ -# Using SGLang as Backend - -Lighteval allows you to use SGLang as a backend, providing significant speedups for model evaluation. -To use SGLang, simply change the `model_args` to reflect the arguments you want to pass to SGLang. - -## Basic Usage - -```bash -lighteval sglang \ - "model_name=HuggingFaceH4/zephyr-7b-beta,dtype=float16" \ - truthfulqa:mc -``` - -## Parallelism Options - -SGLang can distribute the model across multiple GPUs using data parallelism and tensor parallelism. -You can choose the parallelism method by setting the appropriate parameters in the `model_args`. - -### Tensor Parallelism - -For example, if you have 4 GPUs, you can split the model across them using tensor parallelism with `tp_size`: - -```bash -lighteval sglang \ - "model_name=HuggingFaceH4/zephyr-7b-beta,dtype=float16,tp_size=4" \ - truthfulqa:mc -``` - -### Data Parallelism - -If your model fits on a single GPU, you can use data parallelism with `dp_size` to speed up the evaluation: - -```bash -lighteval sglang \ - "model_name=HuggingFaceH4/zephyr-7b-beta,dtype=float16,dp_size=4" \ - truthfulqa:mc -``` - -## Using a Configuration File - -For more advanced configurations, you can use a YAML configuration file for the model. -An example configuration file is shown below and can be found at `examples/model_configs/sglang_model_config.yaml`. - -```bash -lighteval sglang \ - "examples/model_configs/sglang_model_config.yaml" \ - truthfulqa:mc -``` - -> [!TIP] -> Documentation for SGLang server arguments can be found [here](https://docs.sglang.ai/backend/server_arguments.html) - -```yaml -model_parameters: - model_name: "HuggingFaceTB/SmolLM-1.7B-Instruct" - dtype: "auto" - tp_size: 1 - dp_size: 1 - context_length: null - random_seed: 1 - trust_remote_code: False - device: "cuda" - skip_tokenizer_init: False - kv_cache_dtype: "auto" - add_special_tokens: True - pairwise_tokenization: False - sampling_backend: null - attention_backend: null - mem_fraction_static: 0.8 - chunked_prefill_size: 4096 - generation_parameters: - max_new_tokens: 1024 - min_new_tokens: 0 - temperature: 1.0 - top_k: 50 - min_p: 0.0 - top_p: 1.0 - presence_penalty: 0.0 - repetition_penalty: 1.0 - frequency_penalty: 0.0 -``` - -> [!WARNING] -> In case of out-of-memory (OOM) issues, you might need to reduce the context size of the -> model as well as reduce the `mem_fraction_static` and `chunked_prefill_size` parameters. - -## Key SGLang Parameters - -### Memory Management -- `mem_fraction_static`: Fraction of GPU memory to allocate for static tensors (default: 0.8) -- `chunked_prefill_size`: Size of chunks for prefill operations (default: 4096) -- `context_length`: Maximum context length for the model -- `kv_cache_dtype`: Data type for key-value cache - -### Parallelism Settings -- `tp_size`: Number of GPUs for tensor parallelism -- `dp_size`: Number of GPUs for data parallelism - -### Model Configuration -- `dtype`: Data type for model weights ("auto", "float16", "bfloat16", etc.) -- `device`: Device to run the model on ("cuda", "cpu") -- `trust_remote_code`: Whether to trust remote code from the model -- `skip_tokenizer_init`: Skip tokenizer initialization for faster startup - -### Generation Parameters -- `temperature`: Controls randomness in generation (0.0 = deterministic, 1.0 = random) -- `top_p`: Nucleus sampling parameter -- `top_k`: Top-k sampling parameter -- `max_new_tokens`: Maximum number of tokens to generate -- `repetition_penalty`: Penalty for repeating tokens -- `presence_penalty`: Penalty for token presence -- `frequency_penalty`: Penalty for token frequency diff --git a/docs/source/use-vllm-as-backend.mdx b/docs/source/use-vllm-as-backend.mdx deleted file mode 100644 index bf4849754..000000000 --- a/docs/source/use-vllm-as-backend.mdx +++ /dev/null @@ -1,114 +0,0 @@ -# Using VLLM as Backend - -Lighteval allows you to use VLLM as a backend, providing significant speedups for model evaluation. -To use VLLM, simply change the `model_args` to reflect the arguments you want to pass to VLLM. - -> [!TIP] -> Documentation for VLLM engine arguments can be found [here](https://docs.vllm.ai/en/latest/serving/engine_args.html) - -## Basic Usage - -```bash -lighteval vllm \ - "model_name=HuggingFaceH4/zephyr-7b-beta" \ - ifeval -``` - -## Parallelism Options - -VLLM can distribute the model across multiple GPUs using data parallelism, pipeline parallelism, or tensor parallelism. -You can choose the parallelism method by setting the appropriate parameters in the `model_args`. - -### Tensor Parallelism - -For example, if you have 4 GPUs, you can split the model across them using tensor parallelism: - -```bash -export VLLM_WORKER_MULTIPROC_METHOD=spawn && lighteval vllm \ - "model_name=HuggingFaceH4/zephyr-7b-beta,tensor_parallel_size=4" \ - ifeval -``` - -### Data Parallelism - -If your model fits on a single GPU, you can use data parallelism to speed up the evaluation: - -```bash -export VLLM_WORKER_MULTIPROC_METHOD=spawn && lighteval vllm \ - "model_name=HuggingFaceH4/zephyr-7b-beta,data_parallel_size=4" \ - ifeval -``` - -## Using a Configuration File - -For more advanced configurations, you can use a YAML configuration file for the model. -An example configuration file is shown below and can be found at `examples/model_configs/vllm_model_config.yaml`. - -```bash -lighteval vllm \ - "examples/model_configs/vllm_model_config.yaml" \ - ifeval -``` - -```yaml -model_parameters: - model_name: "HuggingFaceTB/SmolLM-1.7B-Instruct" - revision: "main" - dtype: "bfloat16" - tensor_parallel_size: 1 - data_parallel_size: 1 - pipeline_parallel_size: 1 - gpu_memory_utilization: 0.9 - max_model_length: 2048 - swap_space: 4 - seed: 1 - trust_remote_code: True - add_special_tokens: True - multichoice_continuations_start_space: True - pairwise_tokenization: True - subfolder: null - generation_parameters: - presence_penalty: 0.0 - repetition_penalty: 1.0 - frequency_penalty: 0.0 - temperature: 1.0 - top_k: 50 - min_p: 0.0 - top_p: 1.0 - seed: 42 - stop_tokens: null - max_new_tokens: 1024 - min_new_tokens: 0 -``` - -> [!WARNING] -> In case of out-of-memory (OOM) issues, you might need to reduce the context size of the -> model as well as reduce the `gpu_memory_utilization` parameter. - - -## Key VLLM Parameters - -### Memory Management -- `gpu_memory_utilization`: Controls how much GPU memory VLLM can use (default: 0.9) -- `max_model_length`: Maximum sequence length for the model -- `swap_space`: Amount of CPU memory to use for swapping (in GB) - -### Parallelism Settings -- `tensor_parallel_size`: Number of GPUs for tensor parallelism -- `data_parallel_size`: Number of GPUs for data parallelism -- `pipeline_parallel_size`: Number of GPUs for pipeline parallelism - -### Generation Parameters -- `temperature`: Controls randomness in generation (0.0 = deterministic, 1.0 = random) -- `top_p`: Nucleus sampling parameter -- `top_k`: Top-k sampling parameter -- `max_new_tokens`: Maximum number of tokens to generate -- `repetition_penalty`: Penalty for repeating tokens - -## Troubleshooting - -### Common Issues - -1. **Out of Memory Errors**: Reduce `gpu_memory_utilization` or `max_model_length` -2. **Worker Process Issues**: Ensure `VLLM_WORKER_MULTIPROC_METHOD=spawn` is set for multi-GPU setups -3. **Model Loading Errors**: Check that the model name and revision are correct diff --git a/docs/source/using-the-python-api.mdx b/docs/source/using-the-python-api.mdx deleted file mode 100644 index c9bbbbebe..000000000 --- a/docs/source/using-the-python-api.mdx +++ /dev/null @@ -1,103 +0,0 @@ -# Using the Python API - -Lighteval can be used from a custom Python script. To evaluate a model, you will need to set up an -[`~logging.evaluation_tracker.EvaluationTracker`], [`~pipeline.PipelineParameters`], -a [`model`](package_reference/models) or a [`model_config`](package_reference/model_config), -and a [`~pipeline.Pipeline`]. - -After that, simply run the pipeline and save the results. - -```python -import lighteval -from lighteval.logging.evaluation_tracker import EvaluationTracker -from lighteval.models.vllm.vllm_model import VLLMModelConfig -from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters -from lighteval.utils.imports import is_package_available - -if is_package_available("accelerate"): - from datetime import timedelta - from accelerate import Accelerator, InitProcessGroupKwargs - accelerator = Accelerator(kwargs_handlers=[InitProcessGroupKwargs(timeout=timedelta(seconds=3000))]) -else: - accelerator = None - -def main(): - evaluation_tracker = EvaluationTracker( - output_dir="./results", - save_details=True, - push_to_hub=True, - hub_results_org="your_username", # Replace with your actual username - ) - - pipeline_params = PipelineParameters( - launcher_type=ParallelismManager.ACCELERATE, - custom_tasks_directory=None, # Set to path if using custom tasks - # Remove the parameter below once your configuration is tested - max_samples=10 - ) - - model_config = VLLMModelConfig( - model_name="HuggingFaceH4/zephyr-7b-beta", - dtype="float16", - ) - - task = "gsm8k|5" - - pipeline = Pipeline( - tasks=task, - pipeline_parameters=pipeline_params, - evaluation_tracker=evaluation_tracker, - model_config=model_config, - ) - - pipeline.evaluate() - pipeline.save_and_push_results() - pipeline.show_results() - -if __name__ == "__main__": - main() -``` - -## Key Components - -### EvaluationTracker -The `EvaluationTracker` handles logging and saving evaluation results. It can save results locally and optionally push them to the Hugging Face Hub. - -### PipelineParameters -`PipelineParameters` configures how the evaluation pipeline runs, including parallelism settings and task configuration. - -### Model Configuration -Model configurations define the model to be evaluated, including the model name, data type, and other model-specific parameters. Different backends (VLLM, Transformers, etc.) have their own configuration classes. - -### Pipeline -The `Pipeline` orchestrates the entire evaluation process, taking the tasks, model configuration, and parameters to run the evaluation. - -## Running Multiple Tasks - -You can evaluate on multiple tasks by providing a comma-separated list or a file path: - -```python -# Multiple tasks as comma-separated string -tasks = "aime24,aime25" - -# Or load from a file -tasks = "./path/to/tasks.txt" - -pipeline = Pipeline( - tasks=tasks, - # ... other parameters -) -``` - -## Custom Tasks - -To use custom tasks, set the `custom_tasks_directory` parameter to the path containing your custom task definitions: - -```python -pipeline_params = PipelineParameters( - custom_tasks_directory="./path/to/custom/tasks", - # ... other parameters -) -``` - -For more information on creating custom tasks, see the [Adding a Custom Task](adding-a-custom-task) guide. diff --git a/examples/custom_models/google_translate_model.py b/examples/custom_models/google_translate_model.py deleted file mode 100644 index 1fe456900..000000000 --- a/examples/custom_models/google_translate_model.py +++ /dev/null @@ -1,177 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import hashlib -import logging -import os -import time - -import diskcache -import tenacity -from deep_translator import GoogleTranslator -from tqdm import tqdm -from transformers import AutoTokenizer - -from lighteval.data import GenerativeTaskDataset -from lighteval.models.abstract_model import LightevalModel -from lighteval.models.model_output import ( - ModelResponse, -) -from lighteval.tasks.requests import ( - Doc, -) - - -logger = logging.getLogger(__name__) - - -class GoogleTranslateClient(LightevalModel): - def __init__(self, config) -> None: - self.model = config.model_name - self.model_definition_file_path = config.model_definition_file_path - self.config = config - - self._tokenizer = AutoTokenizer.from_pretrained("gpt2") # Use a dummy tokenizer for compatibility - - # Deep-translator also supports other translators - self.translator = GoogleTranslator() - - # Initialize disk cache - cache_dir = os.path.join(os.getcwd(), ".translation_cache") - self.cache = diskcache.Cache(cache_dir) - - self.max_retries = 3 - self.retry_delay = 1 - - def _get_cache_key(self, context: str, src_lang: str, tgt_lang: str) -> str: - """Generate a unique cache key for the translation request.""" - # IMPORTANT: In case we want to support other translators, we can add the translator name to the key - key_string = f"{context}|{src_lang}|{tgt_lang}" - return hashlib.md5(key_string.encode()).hexdigest() - - @tenacity.retry( - stop=tenacity.stop_after_attempt(3), - wait=tenacity.wait_exponential(multiplier=1, min=4, max=10), - retry=tenacity.retry_if_exception_type((Exception)), - before_sleep=lambda retry_state: time.sleep(1), - ) - def _translate_with_cache(self, context: str, src_lang: str, tgt_lang: str) -> str: - """Translate text using cache if available, otherwise call Google Translate with retry logic.""" - cache_key = self._get_cache_key(context, src_lang, tgt_lang) - - # Try to get from cache - if cache_key in self.cache: - result = self.cache[cache_key] - if result is not None and result != "": - return result - logger.warning("Translation in cache is empty. Removing from cache and retrying...") - del self.cache[cache_key] - - try: - # Updated translation call for deep-translator - self.translator.source = src_lang - self.translator.target = tgt_lang - result = self.translator.translate(context) - if result is None or result == "": - result = "" - - self.cache[cache_key] = result - return result - except Exception as e: - logger.warning(f"Translation error: {str(e)}. Retrying...") - raise # Let tenacity handle the retry - - def greedy_until( - self, - requests: list[Doc], - ) -> list[ModelResponse]: - """ - Generates responses using a greedy decoding strategy until certain ending conditions are met. - Results are cached to disk to avoid repeated translations. - - Args: - requests (list[Request]): list of requests containing the context and ending conditions. - - Returns: - list[ModelResponse]: list of generated responses. - """ - for request in requests: - request.tokenized_context = self.tok_encode(request.context) - - dataset = GenerativeTaskDataset(requests=requests, num_dataset_splits=self.DATASET_SPLITS) - results = [] - - for split in tqdm( - dataset.splits_iterator(), - total=dataset.num_dataset_splits, - desc="Splits", - position=0, - disable=False, # self.disable_tqdm, - ): - for r in tqdm(split, desc="Batch", position=1, disable=False): - # Extract source and target languages from task name - # Format is like "community|sdst-text_level:de-fr|0" - src_lang, tgt_lang = r.task_name.split("|")[1].split(":")[-1].split("-") - - context = r.context.replace(f"{src_lang.upper()}: ", "").replace(f"\n{tgt_lang.upper()}: ", "") - result = self._translate_with_cache(context, src_lang, tgt_lang) - if result is None: - result = "" # Set to empty string to prevent errors in metric computation - - cur_response = ModelResponse( - result=result, - logits=None, - generated_tokens=[], - input_tokens=[], - ) - results.append(cur_response) - - return dataset.get_original_order(results) - - @property - def tokenizer(self): - return self._tokenizer - - def tok_encode(self, text: str): - return text - - @property - def add_special_tokens(self) -> bool: - return False - - @property - def max_length(self) -> int: - """Return the maximum sequence length of the model.""" - return 4096 - - def loglikelihood(self, requests: list[Doc]) -> list[ModelResponse]: - """Tokenize the context and continuation and compute the log likelihood of those - tokenized sequences. - """ - raise NotImplementedError - - def loglikelihood_rolling( - self, - requests: list[Doc], - ) -> list[ModelResponse]: - """This function is used to compute the log likelihood of the context for perplexity metrics.""" - raise NotImplementedError diff --git a/examples/custom_models/local_mt_model.py b/examples/custom_models/local_mt_model.py deleted file mode 100644 index 5d74aa78c..000000000 --- a/examples/custom_models/local_mt_model.py +++ /dev/null @@ -1,262 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import logging -from typing import Optional - -import pycountry -import torch -from tqdm import tqdm -from transformers import ( - AutoModelForSeq2SeqLM, - AutoProcessor, - AutoTokenizer, - SeamlessM4Tv2ForTextToText, -) - -from lighteval.data import GenerativeTaskDataset -from lighteval.models.abstract_model import LightevalModel, TokenSequence -from lighteval.models.model_output import ( - ModelResponse, -) -from lighteval.tasks.requests import ( - Doc, -) - - -logger = logging.getLogger(__name__) - - -class LocalMTClient(LightevalModel): - """ - A custom model implementation for local machine translation models, specifically supporting: - - SeamlessM4T v2 models from Meta - - MADLAD-400 models from Google - - This class provides a unified interface for both model families while handling their different - tokenization and generation approaches transparently. - - Args: - config (CustomModelConfig): Configuration containing: - - model (str): Model identifier/path (e.g. "facebook/seamless-m4t-v2-large" or "google/madlad400-7b-mt") - - model_definition_file_path (str): Path to this model definition file - env_config: Environment configuration (unused) - - The model automatically detects whether to load SeamlessM4T or MADLAD based on the model identifier - and initializes the appropriate tokenizer and model. - - Translation tasks should specify the source and target languages in the format: - "{task_name}|{...}:{src}-{tgt}" - where src and tgt are ISO language codes (2 or 3 letter codes supported). - - Example: - ```lighteval custom facebook/seamless-m4t-v2-large examples/custom_models/local_mt_model.py "lighteval|wmt20:fr-de|0" --max-samples 10 --save-details - ``` - - Note: - - SeamlessM4T models use the AutoProcessor for tokenization - - MADLAD models use the standard AutoTokenizer - - Language codes are automatically converted to 3-letter ISO codes for SeamlessM4T - """ - - def __init__(self, config, env_config) -> None: - self.model = config.model - self.model_definition_file_path = config.model_definition_file_path - self.batch_size = 32 - self.device = "cuda" if torch.cuda.is_available() else "cpu" - self.config = config - - # Update model initialization to handle both models - if "seamless-m4t" in config.model: - self._tokenizer = AutoProcessor.from_pretrained(config.model) - self._model = SeamlessM4Tv2ForTextToText.from_pretrained(config.model) - self.model_type = "seamless-4mt" - self.batch_size = 1 - logger.info( - "Using batch size of 1 for seamless-4mt model because it the target language needs to be set for the entire batch." - ) - elif "madlad400" in config.model: - self._tokenizer = AutoTokenizer.from_pretrained(config.model) - self._model = AutoModelForSeq2SeqLM.from_pretrained(config.model) - self.model_type = "madlad400" - else: - raise ValueError(f"Unsupported model: {config.model}") - - self._model.to(self.device) - self._model.eval() - - def _convert_to_iso3(self, lang_code: str) -> str: - """Convert 2-letter ISO code to 3-letter ISO code.""" - try: - return pycountry.languages.get(alpha_2=lang_code.lower()).alpha_3 - except AttributeError: - # If conversion fails, return the original code - return lang_code - - def greedy_until( - self, - requests: list[Doc], - override_bs: Optional[int] = None, - ) -> list[ModelResponse]: - """ - Generates responses using a greedy decoding strategy until certain ending conditions are met. - Results are cached to disk to avoid repeated translations. - - Args: - requests (list[Request]): list of requests containing the context and ending conditions. - override_bs (int, optional): Override the batch size for generation. Defaults to None. - - Returns: - list[ModelResponse]: list of generated responses. - """ - - def get_langs(task_name: str) -> tuple[str, str]: - src, tgt = task_name.split("|")[1].split(":")[-1].split("-") - if self.model_type == "seamless-4mt": - return self._convert_to_iso3(src), self._convert_to_iso3(tgt) - return src, tgt - - # Prepare all inputs first for creating the GenerativeTaskDataset - prepared_requests = [] - for request in requests: - src_lang, tgt_lang = get_langs(request.task_name) - request.context = request.context.replace(f"{src_lang.upper()}: ", "").replace( - f"\n{tgt_lang.upper()}: ", "" - ) - if self.model_type == "madlad400": - request.context = f"<2{tgt_lang}> {request.context}" - - request.tokenized_context = self.tok_encode(request.context) - prepared_requests.append(request) - - # Create dataset after preparation - dataset = GenerativeTaskDataset(requests=prepared_requests, num_dataset_splits=self.DATASET_SPLITS) - results = [] - batch_size = override_bs or self.batch_size - - for split_start, split_end in tqdm( - dataset.splits_start_end_iterator(), - total=dataset.num_dataset_splits, - desc="Splits", - position=0, - disable=False, - ): - # Get all requests for this split directly from sorted_data - current_requests = dataset.sorted_data[split_start:split_end] - - # Process in batches - for batch_idx in tqdm( - range(0, len(current_requests), batch_size), desc="Batches", position=1, disable=False - ): - batch = current_requests[batch_idx : batch_idx + batch_size] - - # Batch tokenize all inputs together instead of concatenating pre-tokenized inputs because of the padding - batch_texts = [r.context for r in batch] - - # This is the tokenization step that really counts, as it actually gets used - tokenizer_kwargs = {"text": batch_texts, "return_tensors": "pt", "padding": True} - if self.model_type == "seamless-4mt": - src_lang = get_langs(batch[0].task_name)[0] - tokenizer_kwargs["src_lang"] = src_lang - - input_ids, attention_mask = self._tokenizer(**tokenizer_kwargs).to(self.device).values() - - generation_sizes = [r.generation_size for r in batch] - assert set(generation_sizes) == {generation_sizes[0]}, "All generation sizes must be the same" - - # Use unpacked values directly - generate_kwargs = { - "input_ids": input_ids, - "attention_mask": attention_mask, - "max_new_tokens": generation_sizes[0], - } - if self.model_type == "seamless-4mt": - tgt_lang = get_langs(batch[0].task_name)[1] - generate_kwargs["tgt_lang"] = tgt_lang - - output_ids = self._model.generate(**generate_kwargs) - translations = self._tokenizer.batch_decode(output_ids, skip_special_tokens=True) - - # Create responses for the batch - for input_tokens, output_tokens, translation in zip(input_ids, output_ids, translations): - results.append( - ModelResponse( - input_tokens=input_tokens, - generated_tokens=output_tokens, - result=translation, - logits=None, - ) - ) - - return dataset.get_original_order(results) - - def cleanup(self): - import gc - - logger.info("Cleaning up GPU memory for local MT client.") - - # Show GPU memory before cleanup - if torch.cuda.is_available(): - logger.info(f"GPU memory before cleanup: {torch.cuda.memory_allocated() / 1024**2:.2f} MB") - - # Delete model and move to CPU - if hasattr(self, "_model"): - self._model.cpu() - del self._model - self._model = None - - if hasattr(self, "_tokenizer"): - del self._tokenizer - self._tokenizer = None - - torch.cuda.empty_cache() - gc.collect() - - # Show GPU memory after cleanup - if torch.cuda.is_available(): - logger.info(f"GPU memory after cleanup: {torch.cuda.memory_allocated() / 1024**2:.2f} MB") - - @property - def tokenizer(self): - return self._tokenizer - - def tok_encode(self, str_to_encode: str | list[str], add_special_tokens: Optional[bool] = None) -> TokenSequence: - return self._tokenizer(text=str_to_encode, add_special_tokens=add_special_tokens or False).to(self.device) - - @property - def add_special_tokens(self) -> bool: - return False - - @property - def max_length(self) -> int: - """Return the maximum sequence length of the model.""" - return 4096 - - def loglikelihood(self, requests: list[Doc], override_bs: Optional[int] = None) -> list[ModelResponse]: - """Tokenize the context and continuation and compute the log likelihood of those - tokenized sequences. - """ - raise NotImplementedError - - def loglikelihood_rolling(self, requests: list[Doc], override_bs: Optional[int] = None) -> list[ModelResponse]: - """This function is used to compute the log likelihood of the context for perplexity metrics.""" - raise NotImplementedError diff --git a/examples/custom_tasks_templates/custom_yourbench_task.py b/examples/custom_tasks_templates/custom_yourbench_task.py deleted file mode 100644 index ee6383fb9..000000000 --- a/examples/custom_tasks_templates/custom_yourbench_task.py +++ /dev/null @@ -1,264 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -import logging -import re - -import numpy as np -from aenum import extend_enum - -from lighteval.metrics.metrics import Metrics -from lighteval.metrics.metrics_sample import JudgeLLM -from lighteval.metrics.utils.metric_utils import CorpusLevelMetricGrouping -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc, SamplingMethod - - -logger = logging.getLogger(__name__) - -JUDGE_ANSWER_SYSTEM_PROMPT = """You will be provided with the summary of a document, a piece of text, a question generated from that text, and the correct or "gold" answer to the question. Additionally, you will receive a model answer. Your task is to determine wether the model answer is correct using the provided "gold" answer as a reference. - -# Steps - -1. **Document Understanding**: - - Analyze the provided document summary to grasp the context and main themes. - -2. **Chunk Understanding**: - - Examine the provided text (chunk) to understand its content. - -3. **Question Understanding**: - - Interpret the given question to fully comprehend what is being asked. - -4. **Ground Truth Answer Understanding**: - - Understand the provided ground truth answer, identifying its key points. - -5. **Answer Understanding**: - - Examine the Model Answer, identifying key points and assessing accuracy and factuality. - -6. **Final Answer**: - - 0 or 1 (0 if the model answer is incorrect, 1 if it is correct). - -# Output Format - -- Provide your final evaluation of whether the answer is correct within `` XML tags. -- Include a detailed analysis for each part within the designated XML tags: ``, ``, ``, ``, ``, and ``. - -# Examples - -**Input**: -```xml - -[Summary] - - - -[Text] - - - -[Question] - - - -[Gold Answer] - - - -[Model Answer] - -``` -**Output**: -```xml - - -Understanding of the summary including key themes - - - -Analysis of the piece of text - - - -Comprehension of the question being asked - - - -Key points from the gold answer - - - -Key points and accuracy of Answer A - - - -1 or 0 (1 if the model answer is correct, 0 if it is incorrect) - -``` - -# Notes - -- Always focus on key points and factual correctness as per the ground truth. -- Avoid any biases and rely solely on the evidence presented. -- Enclose all evaluations and analyses in the specified XML tags for clarity and structure.""" - - -JUDGE_ANSWER_USER_PROMPT = """ -{summary} - - - -{chunk} - - - -{question} - - - -{oracle_answer} - - - -{model_answer} -""" - - -def get_judge_prompt(question: str, answer: str, gold: str, **kwargs): - chunk = kwargs.get("chunks", "") - summary = kwargs.get("documents", "") - - prompt = [ - {"role": "system", "content": JUDGE_ANSWER_SYSTEM_PROMPT}, - { - "role": "user", - "content": JUDGE_ANSWER_USER_PROMPT.format( - summary=summary, chunk=chunk, question=question, oracle_answer=gold, model_answer=answer - ), - }, - ] - - return prompt - - -def process_judge_response_yourbench(response): - # extract the final answer using regex from the response xml - try: - answer = re.search(r"(.*?)", response, re.DOTALL).group(1) - return int(answer) - except Exception as e: - logger.error(f"Error processing judge response: {e}") - return 0 - - -class JudgeLLMYourBench(JudgeLLM): - def __init__(self): - super().__init__( - judge_model_name="gpt-4o-2024-08-06", - template=get_judge_prompt, - process_judge_response=process_judge_response_yourbench, - judge_backend="openai", - short_judge_name="yourbench_judge", - ) - - def compute(self, sample_ids: list[str], responses: list, formatted_docs: list[Doc]) -> list[dict[str, float]]: - # If we are evaluating a multiturn task, we need to have specific field in the formatted doc - questions = [formatted_doc.specific["question"] for formatted_doc in formatted_docs] - golds = [formatted_doc.get_golds()[0] for formatted_doc in formatted_docs] - predictions = [response[0].result[0] for response in responses] - options = [None] * len(questions) - chunks = [formatted_doc.specific["chunks"][0] for formatted_doc in formatted_docs] - documents = [formatted_doc.specific["document"] for formatted_doc in formatted_docs] - - score, _, _ = self.judge.evaluate_answer_batch( - questions, predictions, options, golds, chunks=chunks, documents=documents - ) - - metrics = [] - for i in range(len(sample_ids)): - metrics.append( - { - "accuracy": score[i], - } - ) - - return metrics - - -ZEROSHOT_QA_USER_PROMPT = """Answer the following question: - - -{question} - - -Enclose your full answer in XML tags. For example: - - -[your answer here] -""" - - -def yourbench_prompt(line, task_name: str = ""): - return Doc( - task_name=task_name, - query=ZEROSHOT_QA_USER_PROMPT.format(question=line["question"]), - choices=[line["ground_truth_answer"]], - gold_index=0, - specific={ - "question_category": line["question_category"], - "kind": line["kind"], - "estimated_difficulty": line["estimated_difficulty"], - "document_id": line["document_id"], - "question_generating_model": line["question_generating_model"], - "chunks": line["chunks"], - "question": line["question"], - "document": line["document"], - }, - ) - - -yourbench_metrics = CorpusLevelMetricGrouping( - metric_name=["accuracy"], - higher_is_better={"accuracy": True}, - category=SamplingMethod.GENERATIVE, - sample_level_fn=JudgeLLMYourBench(), - corpus_level_fn={"accuracy": np.mean}, -) -extend_enum(Metrics, "yourbench_metrics", yourbench_metrics) - -yourbench = LightevalTaskConfig( - name="HF_TASK_NAME", # noqa: F821 - prompt_function=yourbench_prompt, - hf_repo="HF_DATASET_NAME", # noqa: F821 - hf_subset="lighteval", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=8192, - metrics=[Metrics.yourbench_metrics], - stop_sequence=[], - version=0, -) - - -TASKS_TABLE = [yourbench] diff --git a/examples/custom_tasks_templates/custom_yourbench_task_from_files.py b/examples/custom_tasks_templates/custom_yourbench_task_from_files.py deleted file mode 100644 index 25d5684a6..000000000 --- a/examples/custom_tasks_templates/custom_yourbench_task_from_files.py +++ /dev/null @@ -1,105 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import json -import logging -import tempfile -from functools import partial -from pathlib import Path - -from custom_yourbench_task_mcq import yourbench_prompt -from datasets import Dataset, DatasetDict - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTask, LightevalTaskConfig - - -logger = logging.getLogger(__name__) - -save_dir = str(tempfile.mkdtemp()) - -ds = DatasetDict( - { - "train": Dataset.from_dict( - { - "question": ["What is 2+2?", "Capital of France?"], - "choices": [["1", "2", "3", "4"], ["Paris", "Berlin", "Rome", "Madrid"]], - "gold": [[3], [0]], - } - ) - } -) - - -CustomTaskConfig = partial( - LightevalTaskConfig, - prompt_function=yourbench_prompt, - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=16, - metrics=[Metrics.gpqa_instruct_metric], - version=0, -) - -# Example 1: save to disk (huggingface format) #### - -ds.save_to_disk(save_dir) - -yourbench_mcq = CustomTaskConfig( - name="tiny_mcqa_dataset", - hf_repo="arrow", - hf_subset="default", - hf_data_files=f"{save_dir}/**/*.arrow", -) - -task = LightevalTask(yourbench_mcq) -eval_docs = task.eval_docs() - -print("\n>>READING TASK FROM ARROW<<") -for doc in eval_docs: - print(doc) - - -# Example 2: jsonlines format #### - -jsonl_path = Path(save_dir) / "train.jsonl" -with open(jsonl_path, "w") as f: - for row in ds["train"]: - f.write(json.dumps(row) + "\n") - -yourbench_mcq = CustomTaskConfig( - name="tiny_mcqa_dataset", - hf_repo="json", - hf_subset="default", - hf_data_files=str(jsonl_path), -) - -task = LightevalTask(yourbench_mcq) -eval_docs = task.eval_docs() - -print("\n>>READING TASK FROM JSONLINES<<") -for doc in eval_docs: - print(doc) - -# TASKS_TABLE = [yourbench_mcq] diff --git a/examples/custom_tasks_templates/custom_yourbench_task_mcq.py b/examples/custom_tasks_templates/custom_yourbench_task_mcq.py deleted file mode 100644 index 3403061d1..000000000 --- a/examples/custom_tasks_templates/custom_yourbench_task_mcq.py +++ /dev/null @@ -1,92 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -import logging - -from aenum import extend_enum - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -logger = logging.getLogger(__name__) - - -ZEROSHOT_QA_INSTRUCTION = """ -Answer the following multiple-choice question by selecting only one letter: A, B, C, or D. Do not explain your answer. -""" - -ZEROSHOT_QA_USER_PROMPT = ( - ZEROSHOT_QA_INSTRUCTION - + """ -Question: {question} - -Choices: -{options} - -Answer: -""" -) - - -def yourbench_prompt(line, task_name: str = ""): - options = "\n".join(f"{chr(65 + i)}. {choice}" for i, choice in enumerate(line["choices"])) - - gold_raw = line["gold"][0] - - if isinstance(gold_raw, str) and gold_raw.strip().isalpha(): - gold_index = ord(gold_raw.strip().upper()) - ord("A") - elif isinstance(gold_raw, int): - gold_index = gold_raw - else: - raise ValueError(f"Unexpected gold label format: {gold_raw!r}") - - return Doc( - instruction=ZEROSHOT_QA_INSTRUCTION, - task_name=task_name, - query=ZEROSHOT_QA_USER_PROMPT.format(question=line["question"], options=options), - choices=line["choices"], - gold_index=gold_index, - ) - - -yourbench_metrics = Metrics.gpqa_instruct_metric - -extend_enum(Metrics, "yourbench_metrics", yourbench_metrics) - -yourbench_mcq = LightevalTaskConfig( - name="HF_TASK_NAME", # noqa: F821 - prompt_function=yourbench_prompt, - hf_repo="HF_DATASET_NAME", # noqa: F821 - hf_subset="lighteval", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=8192, - metrics=[Metrics.yourbench_metrics], - version=0, -) - -TASKS_TABLE = [yourbench_mcq] diff --git a/examples/custom_tasks_tests.py b/examples/custom_tasks_tests.py deleted file mode 100644 index 86c51ee7b..000000000 --- a/examples/custom_tasks_tests.py +++ /dev/null @@ -1,59 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.tasks.gpqa import gpqa_instruct_prompt -from lighteval.tasks.tasks.gsm8k import gsm8k_prompt - - -gsm8k_test = LightevalTaskConfig( - name="gsm8k_test", - prompt_function=gsm8k_prompt, - hf_repo="gsm8k", - hf_subset="main", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling_from_train", - generation_size=512, - metrics=[Metrics.expr_gold_metric], - stop_sequence=None, - version=0, -) - -gpqa_diamond_test = LightevalTaskConfig( - name="gpqa:diamond_test", - prompt_function=gpqa_instruct_prompt, - hf_repo="Idavidrein/gpqa", - hf_subset="gpqa_diamond", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=2048, - metrics=[Metrics.gpqa_instruct_pass_at_k(sample_params={"k": 1})], - stop_sequence=[], # no stop sequence, will use eos token - version=0, -) - -TASKS_TABLE = [gsm8k_test, gpqa_diamond_test] diff --git a/examples/model_configs/endpoint_model.yaml b/examples/model_configs/endpoint_model.yaml deleted file mode 100644 index 0b17a0be9..000000000 --- a/examples/model_configs/endpoint_model.yaml +++ /dev/null @@ -1,22 +0,0 @@ -model_parameters: - reuse_existing: false # if true, ignore all params in instance, and don't delete the endpoint after evaluation - # endpoint_name: "llama-2-7B-lighteval" # needs to be lower case without special characters - - model_name: "meta-llama/Llama-2-7b-hf" - revision: "main" # defaults to "main" - dtype: "float16" # can be any of "awq", "eetq", "gptq", "4bit' or "8bit" (will use bitsandbytes), "bfloat16" or "float16" - accelerator: "gpu" - region: "eu-west-1" - vendor: "aws" - instance_type: "nvidia-a10g" - instance_size: "x1" - framework: "pytorch" - endpoint_type: "protected" - namespace: null # The namespace under which to launch the endpoint. Defaults to the current user's namespace - image_url: null # Optionally specify the docker image to use when launching the endpoint model. E.g., launching models with later releases of the TGI container with support for newer models. - env_vars: - null # Optional environment variables to include when launching the endpoint. e.g., `MAX_INPUT_LENGTH: 2048` - generation_parameters: - max_new_tokens: 256 # maximum number of tokens to generate - temperature: 0.2 - top_p: 0.9 diff --git a/examples/model_configs/inference_providers.yaml b/examples/model_configs/inference_providers.yaml deleted file mode 100644 index 9d8dab81e..000000000 --- a/examples/model_configs/inference_providers.yaml +++ /dev/null @@ -1,8 +0,0 @@ -model_parameters: - model_name: "meta-llama/Llama-3.1-8B-Instruct" - provider: "nebius" - timeout: null - proxies: null - parallel_calls_count: 20 - generation_parameters: - temperature: 0.4 diff --git a/examples/model_configs/litellm_model.yaml b/examples/model_configs/litellm_model.yaml deleted file mode 100644 index b0cb25199..000000000 --- a/examples/model_configs/litellm_model.yaml +++ /dev/null @@ -1,11 +0,0 @@ -model_parameters: - model_name: "openai/deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" - provider: "openai" - base_url: "https://router.huggingface.co/hf-inference/v1" - generation_parameters: - temperature: 0.5 - max_new_tokens: 256 - top_p: 0.9 - seed: 0 - repetition_penalty: 1.0 - frequency_penalty: 0.0 diff --git a/examples/model_configs/peft_model.yaml b/examples/model_configs/peft_model.yaml deleted file mode 100644 index 4a8c011f8..000000000 --- a/examples/model_configs/peft_model.yaml +++ /dev/null @@ -1,14 +0,0 @@ -model_parameters: - model_name: "deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B" # pretrained=model_name,trust_remote_code=boolean,revision=revision_to_use,model_parallel=True ... For a PEFT model, the pretrained model should be the one trained with PEFT and the base model below will contain the original model on which the adapters will be applied. - tokenizer: null # name of tokenizer to use if different from the model's default - subfolder: null # subfolder in the model's directory to use - dtype: "float16" # Specifying the model to be loaded in 4 bit uses BitsAndBytesConfig. The other option is to use "8bit" quantization. - compile: true - revision: "main" # revision to use - trust_remote_code: true # Trust remote code - model_parallel: null # Model parallel - max_length: 2048 # maximum length of the input text and the generated text - - # should go in generation - max_generation_toks: 256 # maximum number of tokens to generate - batch_size: 10 # batch size to use diff --git a/examples/model_configs/quantized_model.yaml b/examples/model_configs/quantized_model.yaml deleted file mode 100644 index 676c92aaf..000000000 --- a/examples/model_configs/quantized_model.yaml +++ /dev/null @@ -1,6 +0,0 @@ -model_parameters: - model_name: "HuggingFaceH4/zephyr-7b-beta" # pretrained=model_name,trust_remote_code=boolean,revision=revision_to_use,model_parallel=True ... - revision: "main" # revision to use - dtype: "4bit" # Specifying the model to be loaded in 4 bit uses BitsAndBytesConfig. The other option is to use "8bit" quantization. - compile: true - batch_size: 1 # batch size to use diff --git a/examples/model_configs/sglang_model_config.yaml b/examples/model_configs/sglang_model_config.yaml deleted file mode 100644 index 7bf3c2626..000000000 --- a/examples/model_configs/sglang_model_config.yaml +++ /dev/null @@ -1,29 +0,0 @@ -model_parameters: - model_name: "HuggingFaceTB/SmolLM-1.7B-Instruct" - dtype: "auto" - tp_size: 1 - dp_size: 1 - context_length: null - random_seed: 1 - trust_remote_code: False - device: "cuda" - skip_tokenizer_init: False - kv_cache_dtype: "auto" - add_special_tokens: True - pairwise_tokenization: False - sampling_backend: null - attention_backend: null - mem_fraction_static: 0.8 - chunked_prefill_size: 4096 - generation_parameters: - max_new_tokens: 1024 - min_new_tokens: 0 - temperature: 1.0 - top_k: 50 - min_p: 0.0 - top_p: 1.0 - presence_penalty: 0.0 - repetition_penalty: 1.0 - frequency_penalty: 0.0 -metrics_options: - yo: null diff --git a/examples/model_configs/tgi_model.yaml b/examples/model_configs/tgi_model.yaml deleted file mode 100644 index a31ba0f7b..000000000 --- a/examples/model_configs/tgi_model.yaml +++ /dev/null @@ -1,6 +0,0 @@ -model_parameters: - inference_server_address: "http://localhost:8080" # Replace with your actual TGI server address - inference_server_auth: null - model_name: null # Optional, only required if the TGI container was launched with model_id pointing to a local directory - generation_parameters: - temperature: 0.1 diff --git a/examples/model_configs/transformers_model.yaml b/examples/model_configs/transformers_model.yaml deleted file mode 100644 index 7b7230174..000000000 --- a/examples/model_configs/transformers_model.yaml +++ /dev/null @@ -1,17 +0,0 @@ -model_parameters: - model_name: "HuggingFaceTB/SmolLM2-1.7B-Instruct" - revision: "57aa3c6599c53705406c648e7acca7e11dc45ea3" - dtype: "float16" - compile: false - model_parallel: false - batch_size: 1 - continuous_batching: false - model_loading_kwargs: - attn_implementation: "eager" - #tp_plan: "auto" - generation_parameters: - #num_blocks: 4096 - #block_size: 64 - #max_new_tokens: 256 - temperature: 0.0 - top_p: 0.9 diff --git a/examples/model_configs/transformers_vlm_model.yaml b/examples/model_configs/transformers_vlm_model.yaml deleted file mode 100644 index 2dc0c6452..000000000 --- a/examples/model_configs/transformers_vlm_model.yaml +++ /dev/null @@ -1,11 +0,0 @@ -model_parameters: - model_name: "Qwen/Qwen2.5-VL-3B-Instruct" - revision: "main" - dtype: "float16" - compile: false - model_parallel: false - batch_size: 1 - use_fast_image_processor: true - generation_parameters: - temperature: 0.0 - top_p: 0.9 diff --git a/examples/model_configs/vllm_model_config.yaml b/examples/model_configs/vllm_model_config.yaml deleted file mode 100644 index 74f0afb92..000000000 --- a/examples/model_configs/vllm_model_config.yaml +++ /dev/null @@ -1,31 +0,0 @@ -model_parameters: - model_name: "HuggingFaceTB/SmolLM2-1.7B-Instruct" - revision: "57aa3c6599c53705406c648e7acca7e11dc45ea3" - dtype: "float16" - tensor_parallel_size: 1 - data_parallel_size: 1 - pipeline_parallel_size: 1 - gpu_memory_utilization: 0.6 - max_model_length: null - swap_space: 4 - seed: 42 - trust_remote_code: False - add_special_tokens: True - multichoice_continuations_start_space: False - pairwise_tokenization: False - subfolder: null - max_num_seqs: 1 - max_num_batched_tokens: 8192 - is_async: false - generation_parameters: - presence_penalty: 0.0 - repetition_penalty: 1.0 - frequency_penalty: 0.0 - temperature: 0.0 - top_k: null - min_p: 0.0 - top_p: 0.9 - seed: 42 - stop_tokens: null - max_new_tokens: 2048 - min_new_tokens: 0 diff --git a/examples/nanotron/custom_evaluation_tasks.py b/examples/nanotron/custom_evaluation_tasks.py deleted file mode 100644 index b079498d8..000000000 --- a/examples/nanotron/custom_evaluation_tasks.py +++ /dev/null @@ -1,712 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -# ruff: noqa: F405, F403, F401 -""" -Custom evaluation tasks for lighteval - -This file generally creates just a TASKS_TABLE and TASKS_GROUPS which are then imported by LightEval. -""" - -import re -from string import ascii_uppercase -from typing import List, Tuple - -from lighteval.metrics.metrics import Metrics -from lighteval.metrics.normalizations import LogProbCharNorm, helm_normalizer, math_normalizer -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc -from lighteval.tasks.tasks.arc import arc_prompt -from lighteval.tasks.tasks.gsm8k import gsm8k_prompt -from lighteval.tasks.tasks.math import math_prompt -from lighteval.tasks.tasks.openbookqa import openbookqa_prompt -from lighteval.tasks.tasks.piqa import piqa_prompt -from lighteval.tasks.tasks.quac import quac_prompt -from lighteval.tasks.tasks.triviaqa import triviaqa_prompt -from lighteval.tasks.tasks.winogrande import winogrande_prompt - - -_TASKS_STRINGS: List[Tuple[LightevalTaskConfig, str]] = [] -_TASKS: List[LightevalTaskConfig] = [] - - -# COMMON_SENSE_REASONING_TASKS ## -def commonsense_qa_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=line["question"], - choices=[f" {c}" for c in line["choices"]["text"]], - gold_index=ascii_uppercase.index(line["answerKey"].strip()), - instruction="", - ) - - -def siqa_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=line["context"] + " " + line["question"], - choices=[f" {c}" for c in [line["answerA"], line["answerB"], line["answerC"]]], - gold_index=int(line["label"]) - 1, - instruction="", - ) - - -def hellaswag_prompt(line, task_name: str = None): - def preprocess(text): - """Comes from AiHarness""" - # text = text.strip() - # NOTE: Brackets are artifacts of the WikiHow dataset portion of HellaSwag. - text = text.replace(" [title]", ". ") - text = re.sub("\\[.*?\\]", "", text) - text = text.replace(" ", " ") - return text - - ctx = f"{line['ctx_a']} {line['ctx_b'].capitalize()} " - return Doc( - task_name=task_name, - query=preprocess(line["activity_label"] + ": " + ctx), - choices=[" " + preprocess(ending) for ending in line["endings"]], - gold_index=int(line["label"]) if line["label"] != "" else -1, # -1 for test - # "metrics": "choices_loglikelihood", - ) - - -COMMON_SENSE_REASONING_TASKS = [ - LightevalTaskConfig( - name="hellaswag", - prompt_function=hellaswag_prompt, - hf_repo="hellaswag", - hf_subset="default", - metrics=[ - Metrics.loglikelihood_acc, - Metrics.loglikelihood_acc( - sample_params={"logprob_normalization": LogProbCharNorm(ignore_first_space=True)} - ), - ], - stop_sequence=["\n"], - ), - LightevalTaskConfig( - name="winogrande", - prompt_function=winogrande_prompt, - hf_repo="winogrande", - hf_subset="winogrande_xl", - metrics=[ - Metrics.loglikelihood_acc, - Metrics.loglikelihood_acc( - sample_params={"logprob_normalization": LogProbCharNorm(ignore_first_space=True)} - ), - ], - stop_sequence=["\n"], - ), - LightevalTaskConfig( - name="piqa", - prompt_function=piqa_prompt, - hf_repo="piqa", - hf_subset="plain_text", - metrics=[ - Metrics.loglikelihood_acc, - Metrics.loglikelihood_acc( - sample_params={"logprob_normalization": LogProbCharNorm(ignore_first_space=True)} - ), - ], - stop_sequence=["\n"], - ), - LightevalTaskConfig( - name="siqa", - prompt_function=siqa_prompt, - hf_repo="lighteval/siqa", - hf_subset="default", - hf_avail_splits=["train", "validation"], - metrics=[ - Metrics.loglikelihood_acc, - Metrics.loglikelihood_acc( - sample_params={"logprob_normalization": LogProbCharNorm(ignore_first_space=True)} - ), - ], - stop_sequence=["\n"], - ), - LightevalTaskConfig( - name="openbookqa", - prompt_function=openbookqa_prompt, - hf_repo="openbookqa", - hf_subset="main", - metrics=[ - Metrics.loglikelihood_acc, - Metrics.loglikelihood_acc( - sample_params={"logprob_normalization": LogProbCharNorm(ignore_first_space=True)} - ), - ], - stop_sequence=["\n"], - ), - LightevalTaskConfig( - name="arc:easy", - prompt_function=arc_prompt, - hf_repo="ai2_arc", - hf_subset="ARC-Easy", - evaluation_splits=["test"], - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - Metrics.loglikelihood_acc( - sample_params={"logprob_normalization": LogProbCharNorm(ignore_first_space=True)} - ), - ], - stop_sequence=["\n"], - ), - LightevalTaskConfig( - name="arc:challenge", - prompt_function=arc_prompt, - hf_repo="ai2_arc", - hf_subset="ARC-Challenge", - evaluation_splits=["test"], - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - Metrics.loglikelihood_acc( - sample_params={"logprob_normalization": LogProbCharNorm(ignore_first_space=True)} - ), - ], - stop_sequence=["\n"], - ), - LightevalTaskConfig( - name="commonsense_qa", - prompt_function=commonsense_qa_prompt, - hf_repo="commonsense_qa", - hf_subset="default", - metrics=[ - Metrics.loglikelihood_acc, - Metrics.loglikelihood_acc( - sample_params={"logprob_normalization": LogProbCharNorm(ignore_first_space=True)} - ), - ], - stop_sequence=["\n"], - ), -] - - -# 0 short for common sense -COMMON_SENSE_REASONING_STRING = [(t, f"custom|{t.name}|0") for t in COMMON_SENSE_REASONING_TASKS] -_TASKS_STRINGS.extend(COMMON_SENSE_REASONING_STRING) -_TASKS += COMMON_SENSE_REASONING_TASKS - - -# WORLD_KNOWLEDGE_TASKS ## -def natural_questions_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=line["question"] + "?\nAnswer: ", - choices=[line["short_answers"]], - gold_index=0, - instruction="", - ) - - -WORLD_KNOWLEDGE_TASKS = [ - LightevalTaskConfig( - name="trivia_qa", - prompt_function=triviaqa_prompt, - hf_repo="trivia_qa", - hf_subset="rc.nocontext", - metrics=[ - Metrics.exact_match(sample_params={"normalize_gold": helm_normalizer, "normalize_pred": helm_normalizer}) - ], - generation_size=20, - stop_sequence=["\n", ".", ","], - ), - LightevalTaskConfig( - name="natural_questions", - prompt_function=natural_questions_prompt, - hf_repo="lighteval/natural_questions_clean", - hf_subset="default", - metrics=[ - Metrics.exact_match(sample_params={"normalize_gold": helm_normalizer, "normalize_pred": helm_normalizer}) - ], - generation_size=20, - stop_sequence=["\n", ".", ","], - ), -] - - -WORLD_KNOWLEDGE_STRING = [(t, f"custom|{t.name}|5") for t in WORLD_KNOWLEDGE_TASKS] -# WORLD_KNOWLEDGE_STRING = {t: f'custom|{t.name}|0' for t in WORLD_KNOWLEDGE_TASKS} -_TASKS_STRINGS.extend(WORLD_KNOWLEDGE_STRING) -_TASKS += WORLD_KNOWLEDGE_TASKS - - -# Reading comprehension ## -def boolq_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"{line['passage']}\nQuestion: {line['question'].capitalize()}?\nAnswer:", - choices=[" No", " Yes"], # Only gold - gold_index=int(line["label"]), - ) - - -READING_COMP_TASKS = [ - LightevalTaskConfig( - name="super_glue:boolq", - prompt_function=boolq_prompt, - hf_repo="super_glue", - hf_subset="boolq", - metrics=[Metrics.target_perplexity], - stop_sequence=["\n"], - ), - LightevalTaskConfig( - name="quac", - prompt_function=quac_prompt, - hf_repo="lighteval/quac_helm", - hf_subset="deault", - metrics=[ - Metrics.exact_match(sample_params={"normalize_gold": helm_normalizer, "normalize_pred": helm_normalizer}) - ], - generation_size=20, - stop_sequence=["\n", ".", ","], - ), -] - - -READING_COMP_STRING = [(t, f"custom|{t.name}|0") for t in READING_COMP_TASKS] -_TASKS_STRINGS.extend(READING_COMP_STRING) -_TASKS += READING_COMP_TASKS - - -# MATH ## -class CustomMathEvaluationTask(LightevalTaskConfig): - """Custom class for math tasks with all the defaults set""" - - def __init__( - self, - name, - prompt_function=math_prompt, - hf_repo="DigitalLearningGmbH/MATH-lighteval", - hf_subset=None, - metrics=[ - Metrics.exact_match(sample_params={"normalize_gold": math_normalizer, "normalize_pred": math_normalizer}) - ], - hf_avail_splits=None, - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=40, - stop_sequence=None, - ): - super().__init__( - name=name, - prompt_function=prompt_function, - hf_repo=hf_repo, - hf_subset=hf_subset, - metrics=metrics, - hf_avail_splits=hf_avail_splits, - evaluation_splits=evaluation_splits, - few_shots_split=few_shots_split, - few_shots_select=few_shots_select, - generation_size=generation_size, - stop_sequence=(stop_sequence if stop_sequence is not None else ["\n"]), - ) - - -MATH_TASKS = [ - CustomMathEvaluationTask(name="math:algebra", hf_subset="algebra"), - CustomMathEvaluationTask(name="math:counting_and_probability", hf_subset="counting_and_probability"), - CustomMathEvaluationTask(name="math:geometry", hf_subset="geometry"), - CustomMathEvaluationTask(name="math:intermediate_algebra", hf_subset="intermediate_algebra"), - CustomMathEvaluationTask(name="math:number_theory", hf_subset="number_theory"), - CustomMathEvaluationTask(name="math:prealgebra", hf_subset="prealgebra"), - CustomMathEvaluationTask(name="math:precalculus", hf_subset="precalculus"), -] -GSM8K = LightevalTaskConfig( - name="gsm8k", - prompt_function=gsm8k_prompt, - hf_repo="gsm8k", - hf_subset="main", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - metrics=[Metrics.perfect_exact_match], - generation_size=10, - stop_sequence=["\n"], -) - - -MATH_STRING = [(t, f"custom|{t.name}|4") for t in MATH_TASKS] -GSM8K_STRING = [(GSM8K, f"custom|{GSM8K.name}|8")] -_TASKS_STRINGS.extend(MATH_STRING) -_TASKS_STRINGS.extend(GSM8K_STRING) -_TASKS += MATH_TASKS + [GSM8K] - - -# MMLU ## -def mmlu_harness(line, task_name: str = None): - topic = line["subject"] - prompt = f"The following are multiple choice questions (with answers) about {topic.replace('_', ' ')}.\n\n" - prompt += line["question"] + "\n" - prompt += "".join([f"{key}. {choice}\n" for key, choice in zip(ascii_uppercase, line["choices"])]) - prompt += "Answer:" - - gold_ix = ascii_uppercase.index(line["answer"]) if isinstance(line["answer"], str) else line["answer"] - "__few_shots" in line and line["__few_shots"] is True # We are adding few shots - - return Doc( - task_name=task_name, - query=prompt, - choices=[" A", " B", " C", " D"], - gold_index=gold_ix, - instruction=f"The following are multiple choice questions (with answers) about {topic.replace('_', ' ')}.\n\n", - ) - - -def mmlu_prompt(line, task_name: str = None): - """MMLU prompt without letters""" - topic = line["subject"] - prompt = f"The following are questions about {topic.replace('_', ' ')}.\nQuestion: " - prompt += line["question"] + "\nAnswer:" - - return Doc( - task_name=task_name, - query=prompt, - choices=[f" {c}" for c in line["choices"]], - gold_index=line["answer"], - instruction=f"The following are questions about {topic.replace('_', ' ')}.\n", - ) - - -class CustomMMLUEvaluationTask(LightevalTaskConfig): - def __init__( - self, - name, - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset=None, - metrics=[ - Metrics.loglikelihood_acc, - Metrics.loglikelihood_acc( - sample_params={"logprob_normalization": LogProbCharNorm(ignore_first_space=True)} - ), - ], - hf_avail_splits=None, - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=-1, - stop_sequence=None, - ): - super().__init__( - name=name, - prompt_function=prompt_function, - hf_repo=hf_repo, - hf_subset=hf_subset, - metrics=metrics, - hf_avail_splits=hf_avail_splits, - evaluation_splits=evaluation_splits, - few_shots_split=few_shots_split, - few_shots_select=few_shots_select, - generation_size=generation_size, - stop_sequence=(stop_sequence if stop_sequence is not None else ["\n"]), - ) - - -MMLU_TASKS = [ - CustomMMLUEvaluationTask(name="mmlu:abstract_algebra", hf_subset="abstract_algebra"), - CustomMMLUEvaluationTask(name="mmlu:anatomy", hf_subset="anatomy"), - CustomMMLUEvaluationTask(name="mmlu:astronomy", hf_subset="astronomy"), - CustomMMLUEvaluationTask(name="mmlu:business_ethics", hf_subset="business_ethics"), - CustomMMLUEvaluationTask(name="mmlu:clinical_knowledge", hf_subset="clinical_knowledge"), - CustomMMLUEvaluationTask(name="mmlu:college_biology", hf_subset="college_biology"), - CustomMMLUEvaluationTask(name="mmlu:college_chemistry", hf_subset="college_chemistry"), - CustomMMLUEvaluationTask(name="mmlu:college_computer_science", hf_subset="college_computer_science"), - CustomMMLUEvaluationTask(name="mmlu:college_mathematics", hf_subset="college_mathematics"), - CustomMMLUEvaluationTask(name="mmlu:college_medicine", hf_subset="college_medicine"), - CustomMMLUEvaluationTask(name="mmlu:college_physics", hf_subset="college_physics"), - CustomMMLUEvaluationTask(name="mmlu:computer_security", hf_subset="computer_security"), - CustomMMLUEvaluationTask(name="mmlu:conceptual_physics", hf_subset="conceptual_physics"), - CustomMMLUEvaluationTask(name="mmlu:econometrics", hf_subset="econometrics"), - CustomMMLUEvaluationTask(name="mmlu:electrical_engineering", hf_subset="electrical_engineering"), - CustomMMLUEvaluationTask(name="mmlu:elementary_mathematics", hf_subset="elementary_mathematics"), - CustomMMLUEvaluationTask(name="mmlu:formal_logic", hf_subset="formal_logic"), - CustomMMLUEvaluationTask(name="mmlu:global_facts", hf_subset="global_facts"), - CustomMMLUEvaluationTask(name="mmlu:high_school_biology", hf_subset="high_school_biology"), - CustomMMLUEvaluationTask(name="mmlu:high_school_chemistry", hf_subset="high_school_chemistry"), - CustomMMLUEvaluationTask(name="mmlu:high_school_computer_science", hf_subset="high_school_computer_science"), - CustomMMLUEvaluationTask(name="mmlu:high_school_european_history", hf_subset="high_school_european_history"), - CustomMMLUEvaluationTask(name="mmlu:high_school_geography", hf_subset="high_school_geography"), - CustomMMLUEvaluationTask( - name="mmlu:high_school_government_and_politics", hf_subset="high_school_government_and_politics" - ), - CustomMMLUEvaluationTask(name="mmlu:high_school_macroeconomics", hf_subset="high_school_macroeconomics"), - CustomMMLUEvaluationTask(name="mmlu:high_school_mathematics", hf_subset="high_school_mathematics"), - CustomMMLUEvaluationTask(name="mmlu:high_school_microeconomics", hf_subset="high_school_microeconomics"), - CustomMMLUEvaluationTask(name="mmlu:high_school_physics", hf_subset="high_school_physics"), - CustomMMLUEvaluationTask(name="mmlu:high_school_psychology", hf_subset="high_school_psychology"), - CustomMMLUEvaluationTask(name="mmlu:high_school_statistics", hf_subset="high_school_statistics"), - CustomMMLUEvaluationTask(name="mmlu:high_school_us_history", hf_subset="high_school_us_history"), - CustomMMLUEvaluationTask(name="mmlu:high_school_world_history", hf_subset="high_school_world_history"), - CustomMMLUEvaluationTask(name="mmlu:human_aging", hf_subset="human_aging"), - CustomMMLUEvaluationTask(name="mmlu:human_sexuality", hf_subset="human_sexuality"), - CustomMMLUEvaluationTask(name="mmlu:international_law", hf_subset="international_law"), - CustomMMLUEvaluationTask(name="mmlu:jurisprudence", hf_subset="jurisprudence"), - CustomMMLUEvaluationTask(name="mmlu:logical_fallacies", hf_subset="logical_fallacies"), - CustomMMLUEvaluationTask(name="mmlu:machine_learning", hf_subset="machine_learning"), - CustomMMLUEvaluationTask(name="mmlu:management", hf_subset="management"), - CustomMMLUEvaluationTask(name="mmlu:marketing", hf_subset="marketing"), - CustomMMLUEvaluationTask(name="mmlu:medical_genetics", hf_subset="medical_genetics"), - CustomMMLUEvaluationTask(name="mmlu:miscellaneous", hf_subset="miscellaneous"), - CustomMMLUEvaluationTask(name="mmlu:moral_disputes", hf_subset="moral_disputes"), - CustomMMLUEvaluationTask(name="mmlu:moral_scenarios", hf_subset="moral_scenarios"), - CustomMMLUEvaluationTask(name="mmlu:nutrition", hf_subset="nutrition"), - CustomMMLUEvaluationTask(name="mmlu:philosophy", hf_subset="philosophy"), - CustomMMLUEvaluationTask(name="mmlu:prehistory", hf_subset="prehistory"), - CustomMMLUEvaluationTask(name="mmlu:professional_accounting", hf_subset="professional_accounting"), - CustomMMLUEvaluationTask(name="mmlu:professional_law", hf_subset="professional_law"), - CustomMMLUEvaluationTask(name="mmlu:professional_medicine", hf_subset="professional_medicine"), - CustomMMLUEvaluationTask(name="mmlu:professional_psychology", hf_subset="professional_psychology"), - CustomMMLUEvaluationTask(name="mmlu:public_relations", hf_subset="public_relations"), - CustomMMLUEvaluationTask(name="mmlu:security_studies", hf_subset="security_studies"), - CustomMMLUEvaluationTask(name="mmlu:sociology", hf_subset="sociology"), - CustomMMLUEvaluationTask(name="mmlu:us_foreign_policy", hf_subset="us_foreign_policy"), - CustomMMLUEvaluationTask(name="mmlu:virology", hf_subset="virology"), - CustomMMLUEvaluationTask(name="mmlu:world_religions", hf_subset="world_religions"), -] - - -# MMLU_STRING = {t: f'custom|{t.name}|5' for t in MMLU_TASKS} -MMLU_STRING = [(t, f"custom|{t.name}|0") for t in MMLU_TASKS] -_TASKS_STRINGS.extend(MMLU_STRING) -_TASKS += MMLU_TASKS - - -# BBH ## -def bbh_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=line["input"] + "\nAnswer: ", - choices=[line["target"]], - gold_index=0, - ) - - -class CustomBBHEvaluationTask(LightevalTaskConfig): - def __init__( - self, - name, - prompt_function=bbh_prompt, - hf_repo="lighteval/big_bench_hard", - hf_subset=None, - metrics=[Metrics.exact_match], - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split="train", - few_shots_select=None, - generation_size=4, - stop_sequence=None, - ): - super().__init__( - name=name, - prompt_function=prompt_function, - hf_repo=hf_repo, - hf_subset=hf_subset, - metrics=metrics, - hf_avail_splits=hf_avail_splits, - evaluation_splits=evaluation_splits, - few_shots_split=few_shots_split, - few_shots_select=few_shots_select, - generation_size=generation_size, - stop_sequence=(stop_sequence if stop_sequence is not None else ["\n"]), - ) - - -BBH_TASKS = [ - CustomBBHEvaluationTask(name="bbh:boolean_expressions", hf_subset="boolean_expressions"), - CustomBBHEvaluationTask(name="bbh:causal_judgement", hf_subset="causal_judgement"), - CustomBBHEvaluationTask(name="bbh:date_understanding", hf_subset="date_understanding"), - CustomBBHEvaluationTask(name="bbh:disambiguation_qa", hf_subset="disambiguation_qa"), - CustomBBHEvaluationTask(name="bbh:dyck_languages", hf_subset="dyck_languages"), - CustomBBHEvaluationTask(name="bbh:formal_fallacies", hf_subset="formal_fallacies"), - CustomBBHEvaluationTask(name="bbh:geometric_shapes", hf_subset="geometric_shapes"), - CustomBBHEvaluationTask(name="bbh:hyperbaton", hf_subset="hyperbaton"), - CustomBBHEvaluationTask(name="bbh:logical_deduction_five_objects", hf_subset="logical_deduction_five_objects"), - CustomBBHEvaluationTask(name="bbh:logical_deduction_seven_objects", hf_subset="logical_deduction_seven_objects"), - CustomBBHEvaluationTask(name="bbh:logical_deduction_three_objects", hf_subset="logical_deduction_three_objects"), - CustomBBHEvaluationTask(name="bbh:movie_recommendation", hf_subset="movie_recommendation"), - CustomBBHEvaluationTask(name="bbh:multistep_arithmetic_two", hf_subset="multistep_arithmetic_two"), - CustomBBHEvaluationTask(name="bbh:navigate", hf_subset="navigate"), - CustomBBHEvaluationTask(name="bbh:object_counting", hf_subset="object_counting"), - CustomBBHEvaluationTask(name="bbh:penguins_in_a_table", hf_subset="penguins_in_a_table"), - CustomBBHEvaluationTask(name="bbh:reasoning_about_colored_objects", hf_subset="reasoning_about_colored_objects"), - CustomBBHEvaluationTask(name="bbh:ruin_names", hf_subset="ruin_names"), - CustomBBHEvaluationTask( - name="bbh:salient_translation_error_detection", hf_subset="salient_translation_error_detection" - ), - CustomBBHEvaluationTask(name="bbh:snarks", hf_subset="snarks"), - CustomBBHEvaluationTask(name="bbh:sports_understanding", hf_subset="sports_understanding"), - CustomBBHEvaluationTask(name="bbh:temporal_sequences", hf_subset="temporal_sequences"), - CustomBBHEvaluationTask( - name="bbh:tracking_shuffled_objects_five_objects", hf_subset="tracking_shuffled_objects_five_objects" - ), - CustomBBHEvaluationTask( - name="bbh:tracking_shuffled_objects_seven_objects", hf_subset="tracking_shuffled_objects_seven_objects" - ), - CustomBBHEvaluationTask( - name="bbh:tracking_shuffled_objects_three_objects", hf_subset="tracking_shuffled_objects_three_objects" - ), - CustomBBHEvaluationTask(name="bbh:web_of_lies", hf_subset="web_of_lies"), - CustomBBHEvaluationTask(name="bbh:word_sorting", hf_subset="word_sorting"), -] - - -# BBH_STRING = {t: f'custom|{t.name}|3' for t in BBH_TASKS} -BBH_STRING = [(t, f"custom|{t.name}|0") for t in BBH_TASKS] -_TASKS_STRINGS.extend(BBH_STRING) -_TASKS += BBH_TASKS - - -# AGI eval ## - - -def agi_eval_math_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=line["question"], - choices=[line["answer"]], - gold_index=0, - instruction="", - ) - - -def agi_eval_prompt(line, task_name: str = None): - cleaned_options = [o.replace("(", "").replace(")", " ") for o in line["options"]] - prompt = "The following are multiple choice questions (with answers).\n\n" - prompt += line["question"] + "\n" + "\n".join(cleaned_options) + "\n" - prompt += "Answer: " - - choices = ascii_uppercase[: len(line["options"])] - - output = Doc( - query=prompt, - instruction="The following are multiple choice questions (with answers).\n\n", - ) - - if line["label"]: - output.choices = choices - output.gold_index = ascii_uppercase.index(line["label"].strip()) - else: - output.choices = [line["answer"]] - output.gold_index = 0 - - return output - - -def agi_eval_prompt_no_letters(line, task_name: str = None): - cleaned_options = [ - " " + o.replace("(A)", "").replace("(B)", "").replace("(C)", "").replace("(D)", "").replace("(E)", "") - for o in line["options"] - ] - - output = Doc( - query=line["question"], - choices=cleaned_options, - gold_index=ascii_uppercase.index(line["label"].strip()), - instruction="", - ) - - return output - - -class CustomAGIEvalEvaluationTask(LightevalTaskConfig): - def __init__( - self, - name, - prompt_function=agi_eval_prompt_no_letters, - hf_repo="lighteval/agi_eval_en", - hf_subset=None, - metrics=[ - Metrics.loglikelihood_acc, - Metrics.loglikelihood_acc( - sample_params={"logprob_normalization": LogProbCharNorm(ignore_first_space=True)} - ), - ], - hf_avail_splits=["train", "validation"], - evaluation_splits=["train"], - few_shots_split="validation", - few_shots_select=None, - generation_size=-1, - stop_sequence=None, - ): - super().__init__( - name=name, - prompt_function=prompt_function, - hf_repo=hf_repo, - hf_subset=hf_subset, - metrics=metrics, - hf_avail_splits=hf_avail_splits, - evaluation_splits=evaluation_splits, - few_shots_split=few_shots_split, - few_shots_select=few_shots_select, - generation_size=generation_size, - stop_sequence=(stop_sequence if stop_sequence is not None else ["\n"]), - ) - - -AGIEVAL_TASKS = [ - CustomAGIEvalEvaluationTask(name="agi_eval:aqua_rat", hf_subset="aqua_rat"), - CustomAGIEvalEvaluationTask(name="agi_eval:logiqa-en", hf_subset="logiqa-en"), - CustomAGIEvalEvaluationTask(name="agi_eval:lsat-ar", hf_subset="lsat-ar"), - CustomAGIEvalEvaluationTask(name="agi_eval:lsat-lr", hf_subset="lsat-lr"), - CustomAGIEvalEvaluationTask(name="agi_eval:lsat-rc", hf_subset="lsat-rc"), - CustomAGIEvalEvaluationTask( - name="agi_eval:math", - hf_subset="math", - prompt_function=agi_eval_math_prompt, - metrics=[ - Metrics.exact_match, - Metrics.exact_match(sample_params={"normalize_gold": helm_normalizer, "normalize_pred": helm_normalizer}), - ], - generation_size=40, - ), - CustomAGIEvalEvaluationTask(name="agi_eval:sat-en", hf_subset="sat-en"), - CustomAGIEvalEvaluationTask(name="agi_eval:sat-math", hf_subset="sat-math"), -] - - -# AGIEVAL_STRING = {t: f'custom|{t.name}|5' for t in AGIEVAL_TASKS} -AGIEVAL_STRING = [(t, f"custom|{t.name}|0") for t in AGIEVAL_TASKS] -_TASKS_STRINGS.extend(AGIEVAL_STRING) -_TASKS += AGIEVAL_TASKS - - -# HUMAN EVAL ## -# human_eval = LightevalTaskConfig( -# name="human_eval", -# prompt_function=prompt.human_eval", -# hf_repo="lighteval/human_eval", -# metrics=["human_eval_pass_at_1"], -# ), - - -EARLY_SIGNAL_TASKS = ",".join([t[1] for t in COMMON_SENSE_REASONING_STRING] + [t[1] for t in MMLU_STRING]) - -# Convert to dict for lighteval -TASKS_TABLE = _TASKS -# You can have a few pre-organised groups of tasks -TASKS_GROUPS = { - "all": ",".join(t[1] for t in _TASKS_STRINGS), - "early-signal": EARLY_SIGNAL_TASKS, -} diff --git a/examples/nanotron/custom_task.py b/examples/nanotron/custom_task.py deleted file mode 100644 index ac55aa4dd..000000000 --- a/examples/nanotron/custom_task.py +++ /dev/null @@ -1,98 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from lighteval.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig - - -def mmlu_signs(line, topic): - prompt = f"The following are multiple choice questions (with answers) about {topic.replace('_', ' ')}.\n\n" - prompt += line["question"] + "\n" - prompt += "".join([f"{key}. {choice}\n" for key, choice in zip(["+", "*", "=", "#"], line["choices"])]) - prompt += "Answer:" - - gold_ix = ["+", "*", "=", "#"].index(line["answer"]) if isinstance(line["answer"], str) else line["answer"] - is_few_shots = line.get("__few_shots", False) # We are adding few shots - - return { - "query": prompt, - "choices": [" +", " *", " =", " #"] if is_few_shots else ["+", "*", "=", "#"], - "fewshot_sorting_class": [" +", " *", " =", " #"][gold_ix], - "gold_index": gold_ix, - "instruction": f"The following are multiple choice questions (with answers) about {topic.replace('_', ' ')}.\n\n", - } - - -def mmlu_anatomy_signs(line): - return mmlu_signs(line, "anatomy") - - -def mmlu_numbers(line, topic): - prompt = f"The following are multiple choice questions (with answers) about {topic.replace('_', ' ')}.\n\n" - prompt += line["question"] + "\n" - prompt += "".join([f"{key}. {choice}\n" for key, choice in zip(["1", "2", "3", "4"], line["choices"])]) - prompt += "Answer:" - - gold_ix = ["1", "2", "3", "4"].index(line["answer"]) if isinstance(line["answer"], str) else line["answer"] - is_few_shots = line.get("__few_shots", False) # We are adding few shots - - return { - "query": prompt, - "choices": [" 1", " 2", " 3", " 4"] if is_few_shots else ["1", "2", "3", "4"], - "fewshot_sorting_class": [" 1", " 2", " 3", " 4"][gold_ix], - "gold_index": gold_ix, - "instruction": f"The following are multiple choice questions (with answers) about {topic.replace('_', ' ')}.\n\n", - } - - -def mmlu_anatomy(line): - return mmlu_numbers(line, "anatomy") - - -TASKS_TABLE = [ - LightevalTaskConfig( - name="mmlu:anatomy", - prompt_function=mmlu_anatomy, - hf_repo="lighteval/mmlu", - hf_subset="anatomy", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select="sequential", - generation_size=5, - metric=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - ), - LightevalTaskConfig( - name="mmlu:anatomy_signs", - prompt_function=mmlu_anatomy_signs, - hf_repo="lighteval/mmlu", - hf_subset="anatomy", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select="sequential", - generation_size=5, - metric=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - ), -] diff --git a/examples/nanotron/lighteval_config_override_template.yaml b/examples/nanotron/lighteval_config_override_template.yaml deleted file mode 100644 index 433498c34..000000000 --- a/examples/nanotron/lighteval_config_override_template.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# As of right now auto batch size doesn't work, so we use some default -batch_size: 8 -generation: null -logging: - output_dir: "outputs" - save_details: false - push_to_hub: false - public_run: false - results_org: null - tensorboard_metric_prefix: "eval" -parallelism: - dp: 1 - pp: 1 - pp_engine: 1f1b - tp: 1 - tp_linear_async_communication: false - tp_mode: ALL_REDUCE -tasks: - dataset_loading_processes: 8 - max_samples: 10 - multichoice_continuations_start_space: null - num_fewshot_seeds: null - tasks: lighteval|gsm8k|5 diff --git a/examples/tasks/OALL_v1_tasks.txt b/examples/tasks/OALL_v1_tasks.txt deleted file mode 100644 index daecb62a7..000000000 --- a/examples/tasks/OALL_v1_tasks.txt +++ /dev/null @@ -1,136 +0,0 @@ -lighteval|xstory_cloze:ar|0 -community|arabic_mmlu_mt:abstract_algebra|0 -community|arabic_mmlu_mt:anatomy|0 -community|arabic_mmlu_mt:astronomy|0 -community|arabic_mmlu_mt:business_ethics|0 -community|arabic_mmlu_mt:clinical_knowledge|0 -community|arabic_mmlu_mt:college_biology|0 -community|arabic_mmlu_mt:college_chemistry|0 -community|arabic_mmlu_mt:college_computer_science|0 -community|arabic_mmlu_mt:college_mathematics|0 -community|arabic_mmlu_mt:college_medicine|0 -community|arabic_mmlu_mt:college_physics|0 -community|arabic_mmlu_mt:computer_security|0 -community|arabic_mmlu_mt:conceptual_physics|0 -community|arabic_mmlu_mt:econometrics|0 -community|arabic_mmlu_mt:electrical_engineering|0 -community|arabic_mmlu_mt:elementary_mathematics|0 -community|arabic_mmlu_mt:formal_logic|0 -community|arabic_mmlu_mt:global_facts|0 -community|arabic_mmlu_mt:high_school_biology|0 -community|arabic_mmlu_mt:high_school_chemistry|0 -community|arabic_mmlu_mt:high_school_computer_science|0 -community|arabic_mmlu_mt:high_school_european_history|0 -community|arabic_mmlu_mt:high_school_geography|0 -community|arabic_mmlu_mt:high_school_government_and_politics|0 -community|arabic_mmlu_mt:high_school_macroeconomics|0 -community|arabic_mmlu_mt:high_school_mathematics|0 -community|arabic_mmlu_mt:high_school_microeconomics|0 -community|arabic_mmlu_mt:high_school_physics|0 -community|arabic_mmlu_mt:high_school_psychology|0 -community|arabic_mmlu_mt:high_school_statistics|0 -community|arabic_mmlu_mt:high_school_us_history|0 -community|arabic_mmlu_mt:high_school_world_history|0 -community|arabic_mmlu_mt:human_aging|0 -community|arabic_mmlu_mt:human_sexuality|0 -community|arabic_mmlu_mt:international_law|0 -community|arabic_mmlu_mt:jurisprudence|0 -community|arabic_mmlu_mt:logical_fallacies|0 -community|arabic_mmlu_mt:machine_learning|0 -community|arabic_mmlu_mt:management|0 -community|arabic_mmlu_mt:marketing|0 -community|arabic_mmlu_mt:medical_genetics|0 -community|arabic_mmlu_mt:miscellaneous|0 -community|arabic_mmlu_mt:moral_disputes|0 -community|arabic_mmlu_mt:moral_scenarios|0 -community|arabic_mmlu_mt:nutrition|0 -community|arabic_mmlu_mt:philosophy|0 -community|arabic_mmlu_mt:prehistory|0 -community|arabic_mmlu_mt:professional_accounting|0 -community|arabic_mmlu_mt:professional_law|0 -community|arabic_mmlu_mt:professional_medicine|0 -community|arabic_mmlu_mt:professional_psychology|0 -community|arabic_mmlu_mt:public_relations|0 -community|arabic_mmlu_mt:security_studies|0 -community|arabic_mmlu_mt:sociology|0 -community|arabic_mmlu_mt:us_foreign_policy|0 -community|arabic_mmlu_mt:virology|0 -community|arabic_mmlu_mt:world_religions|0 -community|arabic_exams|0 -community|acva:Algeria|0 -community|acva:Ancient_Egypt|0 -community|acva:Arab_Empire|0 -community|acva:Arabic_Architecture|0 -community|acva:Arabic_Art|0 -community|acva:Arabic_Astronomy|0 -community|acva:Arabic_Calligraphy|0 -community|acva:Arabic_Ceremony|0 -community|acva:Arabic_Clothing|0 -community|acva:Arabic_Culture|0 -community|acva:Arabic_Food|0 -community|acva:Arabic_Funeral|0 -community|acva:Arabic_Geography|0 -community|acva:Arabic_History|0 -community|acva:Arabic_Language_Origin|0 -community|acva:Arabic_Literature|0 -community|acva:Arabic_Math|0 -community|acva:Arabic_Medicine|0 -community|acva:Arabic_Music|0 -community|acva:Arabic_Ornament|0 -community|acva:Arabic_Philosophy|0 -community|acva:Arabic_Physics_and_Chemistry|0 -community|acva:Arabic_Wedding|0 -community|acva:Bahrain|0 -community|acva:Comoros|0 -community|acva:Egypt_modern|0 -community|acva:InfluenceFromAncientEgypt|0 -community|acva:InfluenceFromByzantium|0 -community|acva:InfluenceFromChina|0 -community|acva:InfluenceFromGreece|0 -community|acva:InfluenceFromIslam|0 -community|acva:InfluenceFromPersia|0 -community|acva:InfluenceFromRome|0 -community|acva:Iraq|0 -community|acva:Islam_Education|0 -community|acva:Islam_branches_and_schools|0 -community|acva:Islamic_law_system|0 -community|acva:Jordan|0 -community|acva:Kuwait|0 -community|acva:Lebanon|0 -community|acva:Libya|0 -community|acva:Mauritania|0 -community|acva:Mesopotamia_civilization|0 -community|acva:Morocco|0 -community|acva:Oman|0 -community|acva:Palestine|0 -community|acva:Qatar|0 -community|acva:Saudi_Arabia|0 -community|acva:Somalia|0 -community|acva:Sudan|0 -community|acva:Syria|0 -community|acva:Tunisia|0 -community|acva:United_Arab_Emirates|0 -community|acva:Yemen|0 -community|acva:communication|0 -community|acva:computer_and_phone|0 -community|acva:daily_life|0 -community|acva:entertainment|0 -community|alghafa:mcq_exams_test_ar|0 -community|alghafa:meta_ar_dialects|0 -community|alghafa:meta_ar_msa|0 -community|alghafa:multiple_choice_facts_truefalse_balanced_task|0 -community|alghafa:multiple_choice_grounded_statement_soqal_task|0 -community|alghafa:multiple_choice_grounded_statement_xglue_mlqa_task|0 -community|alghafa:multiple_choice_rating_sentiment_no_neutral_task|0 -community|alghafa:multiple_choice_rating_sentiment_task|0 -community|alghafa:multiple_choice_sentiment_task|0 -community|race_ar|0 -community|piqa_ar|0 -community|arc_easy_ar|0 -community|arc_challenge_okapi_ar|0 -community|openbook_qa_ext_ar|0 -community|boolq_ar|0 -community|copa_ext_ar|0 -community|hellaswag_okapi_ar|0 -community|toxigen_ar|0 -community|sciq_ar|0 diff --git a/examples/tasks/OALL_v2_tasks.txt b/examples/tasks/OALL_v2_tasks.txt deleted file mode 100644 index 890c551c4..000000000 --- a/examples/tasks/OALL_v2_tasks.txt +++ /dev/null @@ -1,117 +0,0 @@ -community|alghafa:meta_ar_dialects|0 -community|alghafa:meta_ar_msa|0 -community|alghafa:multiple_choice_facts_truefalse_balanced_task|0 -community|alghafa:multiple_choice_grounded_statement_soqal_task|0 -community|alghafa:multiple_choice_grounded_statement_xglue_mlqa_task|0 -community|alghafa:multiple_choice_rating_sentiment_no_neutral_task|0 -community|alghafa:multiple_choice_rating_sentiment_task|0 -community|alghafa:multiple_choice_sentiment_task|0 -community|arabic_exams|0 -community|arabic_mmlu:Islamic Studies|0 -community|arabic_mmlu:Islamic Studies (Middle School)|0 -community|arabic_mmlu:Islamic Studies (Primary School)|0 -community|arabic_mmlu:Islamic Studies (High School)|0 -community|arabic_mmlu:Driving Test|0 -community|arabic_mmlu:Natural Science (Middle School)|0 -community|arabic_mmlu:Natural Science (Primary School)|0 -community|arabic_mmlu:History (Middle School)|0 -community|arabic_mmlu:History (Primary School)|0 -community|arabic_mmlu:History (High School)|0 -community|arabic_mmlu:General Knowledge|0 -community|arabic_mmlu:General Knowledge (Middle School)|0 -community|arabic_mmlu:General Knowledge (Primary School)|0 -community|arabic_mmlu:Law (Professional)|0 -community|arabic_mmlu:Physics (High School)|0 -community|arabic_mmlu:Social Science (Middle School)|0 -community|arabic_mmlu:Social Science (Primary School)|0 -community|arabic_mmlu:Management (University)|0 -community|arabic_mmlu:Arabic Language (Middle School)|0 -community|arabic_mmlu:Arabic Language (Primary School)|0 -community|arabic_mmlu:Arabic Language (High School)|0 -community|arabic_mmlu:Political Science (University)|0 -community|arabic_mmlu:Philosophy (High School)|0 -community|arabic_mmlu:Accounting (University)|0 -community|arabic_mmlu:Computer Science (Middle School)|0 -community|arabic_mmlu:Computer Science (Primary School)|0 -community|arabic_mmlu:Computer Science (High School)|0 -community|arabic_mmlu:Computer Science (University)|0 -community|arabic_mmlu:Geography (Middle School)|0 -community|arabic_mmlu:Geography (Primary School)|0 -community|arabic_mmlu:Geography (High School)|0 -community|arabic_mmlu:Math (Primary School)|0 -community|arabic_mmlu:Biology (High School)|0 -community|arabic_mmlu:Economics (Middle School)|0 -community|arabic_mmlu:Economics (High School)|0 -community|arabic_mmlu:Economics (University)|0 -community|arabic_mmlu:Arabic Language (General)|0 -community|arabic_mmlu:Arabic Language (Grammar)|0 -community|arabic_mmlu:Civics (Middle School)|0 -community|arabic_mmlu:Civics (High School)|0 -community|madinah_qa:Arabic Language (General)|0 -community|madinah_qa:Arabic Language (Grammar)|0 -community|aratrust:Trustfulness|0 -community|aratrust:MentalHealth|0 -community|aratrust:PhysicalHealth|0 -community|aratrust:Offensive|0 -community|aratrust:Ethics|0 -community|aratrust:Privacy|0 -community|aratrust:Unfairness|0 -community|aratrust:Illegal|0 -community|arabic_mmlu_ht:abstract_algebra|0 -community|arabic_mmlu_ht:anatomy|0 -community|arabic_mmlu_ht:astronomy|0 -community|arabic_mmlu_ht:business_ethics|0 -community|arabic_mmlu_ht:clinical_knowledge|0 -community|arabic_mmlu_ht:college_biology|0 -community|arabic_mmlu_ht:college_chemistry|0 -community|arabic_mmlu_ht:college_computer_science|0 -community|arabic_mmlu_ht:college_mathematics|0 -community|arabic_mmlu_ht:college_medicine|0 -community|arabic_mmlu_ht:college_physics|0 -community|arabic_mmlu_ht:computer_security|0 -community|arabic_mmlu_ht:conceptual_physics|0 -community|arabic_mmlu_ht:econometrics|0 -community|arabic_mmlu_ht:electrical_engineering|0 -community|arabic_mmlu_ht:elementary_mathematics|0 -community|arabic_mmlu_ht:formal_logic|0 -community|arabic_mmlu_ht:global_facts|0 -community|arabic_mmlu_ht:high_school_biology|0 -community|arabic_mmlu_ht:high_school_chemistry|0 -community|arabic_mmlu_ht:high_school_computer_science|0 -community|arabic_mmlu_ht:high_school_european_history|0 -community|arabic_mmlu_ht:high_school_geography|0 -community|arabic_mmlu_ht:high_school_government_and_politics|0 -community|arabic_mmlu_ht:high_school_macroeconomics|0 -community|arabic_mmlu_ht:high_school_mathematics|0 -community|arabic_mmlu_ht:high_school_microeconomics|0 -community|arabic_mmlu_ht:high_school_physics|0 -community|arabic_mmlu_ht:high_school_psychology|0 -community|arabic_mmlu_ht:high_school_statistics|0 -community|arabic_mmlu_ht:high_school_us_history|0 -community|arabic_mmlu_ht:high_school_world_history|0 -community|arabic_mmlu_ht:human_aging|0 -community|arabic_mmlu_ht:human_sexuality|0 -community|arabic_mmlu_ht:international_law|0 -community|arabic_mmlu_ht:jurisprudence|0 -community|arabic_mmlu_ht:logical_fallacies|0 -community|arabic_mmlu_ht:machine_learning|0 -community|arabic_mmlu_ht:management|0 -community|arabic_mmlu_ht:marketing|0 -community|arabic_mmlu_ht:medical_genetics|0 -community|arabic_mmlu_ht:miscellaneous|0 -community|arabic_mmlu_ht:moral_disputes|0 -community|arabic_mmlu_ht:moral_scenarios|0 -community|arabic_mmlu_ht:nutrition|0 -community|arabic_mmlu_ht:philosophy|0 -community|arabic_mmlu_ht:prehistory|0 -community|arabic_mmlu_ht:professional_accounting|0 -community|arabic_mmlu_ht:professional_law|0 -community|arabic_mmlu_ht:professional_medicine|0 -community|arabic_mmlu_ht:professional_psychology|0 -community|arabic_mmlu_ht:public_relations|0 -community|arabic_mmlu_ht:security_studies|0 -community|arabic_mmlu_ht:sociology|0 -community|arabic_mmlu_ht:us_foreign_policy|0 -community|arabic_mmlu_ht:virology|0 -community|arabic_mmlu_ht:world_religions|0 -community|alrage_qa|0 diff --git a/examples/tasks/all_arabic_tasks.txt b/examples/tasks/all_arabic_tasks.txt deleted file mode 100644 index f88738993..000000000 --- a/examples/tasks/all_arabic_tasks.txt +++ /dev/null @@ -1,244 +0,0 @@ -lighteval|xstory_cloze:ar|0 -community|arabic_exams|0 -community|arabic_mmlu_mt:abstract_algebra|0 -community|arabic_mmlu_mt:anatomy|0 -community|arabic_mmlu_mt:astronomy|0 -community|arabic_mmlu_mt:business_ethics|0 -community|arabic_mmlu_mt:clinical_knowledge|0 -community|arabic_mmlu_mt:college_biology|0 -community|arabic_mmlu_mt:college_chemistry|0 -community|arabic_mmlu_mt:college_computer_science|0 -community|arabic_mmlu_mt:college_mathematics|0 -community|arabic_mmlu_mt:college_medicine|0 -community|arabic_mmlu_mt:college_physics|0 -community|arabic_mmlu_mt:computer_security|0 -community|arabic_mmlu_mt:conceptual_physics|0 -community|arabic_mmlu_mt:econometrics|0 -community|arabic_mmlu_mt:electrical_engineering|0 -community|arabic_mmlu_mt:elementary_mathematics|0 -community|arabic_mmlu_mt:formal_logic|0 -community|arabic_mmlu_mt:global_facts|0 -community|arabic_mmlu_mt:high_school_biology|0 -community|arabic_mmlu_mt:high_school_chemistry|0 -community|arabic_mmlu_mt:high_school_computer_science|0 -community|arabic_mmlu_mt:high_school_european_history|0 -community|arabic_mmlu_mt:high_school_geography|0 -community|arabic_mmlu_mt:high_school_government_and_politics|0 -community|arabic_mmlu_mt:high_school_macroeconomics|0 -community|arabic_mmlu_mt:high_school_mathematics|0 -community|arabic_mmlu_mt:high_school_microeconomics|0 -community|arabic_mmlu_mt:high_school_physics|0 -community|arabic_mmlu_mt:high_school_psychology|0 -community|arabic_mmlu_mt:high_school_statistics|0 -community|arabic_mmlu_mt:high_school_us_history|0 -community|arabic_mmlu_mt:high_school_world_history|0 -community|arabic_mmlu_mt:human_aging|0 -community|arabic_mmlu_mt:human_sexuality|0 -community|arabic_mmlu_mt:international_law|0 -community|arabic_mmlu_mt:jurisprudence|0 -community|arabic_mmlu_mt:logical_fallacies|0 -community|arabic_mmlu_mt:machine_learning|0 -community|arabic_mmlu_mt:management|0 -community|arabic_mmlu_mt:marketing|0 -community|arabic_mmlu_mt:medical_genetics|0 -community|arabic_mmlu_mt:miscellaneous|0 -community|arabic_mmlu_mt:moral_disputes|0 -community|arabic_mmlu_mt:moral_scenarios|0 -community|arabic_mmlu_mt:nutrition|0 -community|arabic_mmlu_mt:philosophy|0 -community|arabic_mmlu_mt:prehistory|0 -community|arabic_mmlu_mt:professional_accounting|0 -community|arabic_mmlu_mt:professional_law|0 -community|arabic_mmlu_mt:professional_medicine|0 -community|arabic_mmlu_mt:professional_psychology|0 -community|arabic_mmlu_mt:public_relations|0 -community|arabic_mmlu_mt:security_studies|0 -community|arabic_mmlu_mt:sociology|0 -community|arabic_mmlu_mt:us_foreign_policy|0 -community|arabic_mmlu_mt:virology|0 -community|arabic_mmlu_mt:world_religions|0 -community|acva:Algeria|0 -community|acva:Ancient_Egypt|0 -community|acva:Arab_Empire|0 -community|acva:Arabic_Architecture|0 -community|acva:Arabic_Art|0 -community|acva:Arabic_Astronomy|0 -community|acva:Arabic_Calligraphy|0 -community|acva:Arabic_Ceremony|0 -community|acva:Arabic_Clothing|0 -community|acva:Arabic_Culture|0 -community|acva:Arabic_Food|0 -community|acva:Arabic_Funeral|0 -community|acva:Arabic_Geography|0 -community|acva:Arabic_History|0 -community|acva:Arabic_Language_Origin|0 -community|acva:Arabic_Literature|0 -community|acva:Arabic_Math|0 -community|acva:Arabic_Medicine|0 -community|acva:Arabic_Music|0 -community|acva:Arabic_Ornament|0 -community|acva:Arabic_Philosophy|0 -community|acva:Arabic_Physics_and_Chemistry|0 -community|acva:Arabic_Wedding|0 -community|acva:Bahrain|0 -community|acva:Comoros|0 -community|acva:Egypt_modern|0 -community|acva:InfluenceFromAncientEgypt|0 -community|acva:InfluenceFromByzantium|0 -community|acva:InfluenceFromChina|0 -community|acva:InfluenceFromGreece|0 -community|acva:InfluenceFromIslam|0 -community|acva:InfluenceFromPersia|0 -community|acva:InfluenceFromRome|0 -community|acva:Iraq|0 -community|acva:Islam_Education|0 -community|acva:Islam_branches_and_schools|0 -community|acva:Islamic_law_system|0 -community|acva:Jordan|0 -community|acva:Kuwait|0 -community|acva:Lebanon|0 -community|acva:Libya|0 -community|acva:Mauritania|0 -community|acva:Mesopotamia_civilization|0 -community|acva:Morocco|0 -community|acva:Oman|0 -community|acva:Palestine|0 -community|acva:Qatar|0 -community|acva:Saudi_Arabia|0 -community|acva:Somalia|0 -community|acva:Sudan|0 -community|acva:Syria|0 -community|acva:Tunisia|0 -community|acva:United_Arab_Emirates|0 -community|acva:Yemen|0 -community|acva:communication|0 -community|acva:computer_and_phone|0 -community|acva:daily_life|0 -community|acva:entertainment|0 -community|alghafa:mcq_exams_test_ar|0 -community|alghafa:meta_ar_dialects|0 -community|alghafa:meta_ar_msa|0 -community|alghafa:multiple_choice_facts_truefalse_balanced_task|0 -community|alghafa:multiple_choice_grounded_statement_soqal_task|0 -community|alghafa:multiple_choice_grounded_statement_xglue_mlqa_task|0 -community|alghafa:multiple_choice_rating_sentiment_no_neutral_task|0 -community|alghafa:multiple_choice_rating_sentiment_task|0 -community|alghafa:multiple_choice_sentiment_task|0 -community|race_ar|0 -community|piqa_ar|0 -community|arc_easy_ar|0 -community|arc_challenge_okapi_ar|0 -community|mmlu_okapi_ar|0 -community|openbook_qa_ext_ar|0 -community|boolq_ar|0 -community|copa_ext_ar|0 -community|hellaswag_okapi_ar|0 -community|toxigen_ar|0 -community|sciq_ar|0 -community|arabic_mmlu_ht:abstract_algebra|0 -community|arabic_mmlu_ht:anatomy|0 -community|arabic_mmlu_ht:astronomy|0 -community|arabic_mmlu_ht:business_ethics|0 -community|arabic_mmlu_ht:clinical_knowledge|0 -community|arabic_mmlu_ht:college_biology|0 -community|arabic_mmlu_ht:college_chemistry|0 -community|arabic_mmlu_ht:college_computer_science|0 -community|arabic_mmlu_ht:college_mathematics|0 -community|arabic_mmlu_ht:college_medicine|0 -community|arabic_mmlu_ht:college_physics|0 -community|arabic_mmlu_ht:computer_security|0 -community|arabic_mmlu_ht:conceptual_physics|0 -community|arabic_mmlu_ht:econometrics|0 -community|arabic_mmlu_ht:electrical_engineering|0 -community|arabic_mmlu_ht:elementary_mathematics|0 -community|arabic_mmlu_ht:formal_logic|0 -community|arabic_mmlu_ht:global_facts|0 -community|arabic_mmlu_ht:high_school_biology|0 -community|arabic_mmlu_ht:high_school_chemistry|0 -community|arabic_mmlu_ht:high_school_computer_science|0 -community|arabic_mmlu_ht:high_school_european_history|0 -community|arabic_mmlu_ht:high_school_geography|0 -community|arabic_mmlu_ht:high_school_government_and_politics|0 -community|arabic_mmlu_ht:high_school_macroeconomics|0 -community|arabic_mmlu_ht:high_school_mathematics|0 -community|arabic_mmlu_ht:high_school_microeconomics|0 -community|arabic_mmlu_ht:high_school_physics|0 -community|arabic_mmlu_ht:high_school_psychology|0 -community|arabic_mmlu_ht:high_school_statistics|0 -community|arabic_mmlu_ht:high_school_us_history|0 -community|arabic_mmlu_ht:high_school_world_history|0 -community|arabic_mmlu_ht:human_aging|0 -community|arabic_mmlu_ht:human_sexuality|0 -community|arabic_mmlu_ht:international_law|0 -community|arabic_mmlu_ht:jurisprudence|0 -community|arabic_mmlu_ht:logical_fallacies|0 -community|arabic_mmlu_ht:machine_learning|0 -community|arabic_mmlu_ht:management|0 -community|arabic_mmlu_ht:marketing|0 -community|arabic_mmlu_ht:medical_genetics|0 -community|arabic_mmlu_ht:miscellaneous|0 -community|arabic_mmlu_ht:moral_disputes|0 -community|arabic_mmlu_ht:moral_scenarios|0 -community|arabic_mmlu_ht:nutrition|0 -community|arabic_mmlu_ht:philosophy|0 -community|arabic_mmlu_ht:prehistory|0 -community|arabic_mmlu_ht:professional_accounting|0 -community|arabic_mmlu_ht:professional_law|0 -community|arabic_mmlu_ht:professional_medicine|0 -community|arabic_mmlu_ht:professional_psychology|0 -community|arabic_mmlu_ht:public_relations|0 -community|arabic_mmlu_ht:security_studies|0 -community|arabic_mmlu_ht:sociology|0 -community|arabic_mmlu_ht:us_foreign_policy|0 -community|arabic_mmlu_ht:virology|0 -community|arabic_mmlu_ht:world_religions|0 -community|arabic_mmlu:Islamic Studies|0 -community|arabic_mmlu:Islamic Studies (Middle School)|0 -community|arabic_mmlu:Islamic Studies (Primary School)|0 -community|arabic_mmlu:Islamic Studies (High School)|0 -community|arabic_mmlu:Driving Test|0 -community|arabic_mmlu:Natural Science (Middle School)|0 -community|arabic_mmlu:Natural Science (Primary School)|0 -community|arabic_mmlu:History (Middle School)|0 -community|arabic_mmlu:History (Primary School)|0 -community|arabic_mmlu:History (High School)|0 -community|arabic_mmlu:General Knowledge|0 -community|arabic_mmlu:General Knowledge (Middle School)|0 -community|arabic_mmlu:General Knowledge (Primary School)|0 -community|arabic_mmlu:Law (Professional)|0 -community|arabic_mmlu:Physics (High School)|0 -community|arabic_mmlu:Social Science (Middle School)|0 -community|arabic_mmlu:Social Science (Primary School)|0 -community|arabic_mmlu:Management (University)|0 -community|arabic_mmlu:Arabic Language (Middle School)|0 -community|arabic_mmlu:Arabic Language (Primary School)|0 -community|arabic_mmlu:Arabic Language (High School)|0 -community|arabic_mmlu:Political Science (University)|0 -community|arabic_mmlu:Philosophy (High School)|0 -community|arabic_mmlu:Accounting (University)|0 -community|arabic_mmlu:Computer Science (Middle School)|0 -community|arabic_mmlu:Computer Science (Primary School)|0 -community|arabic_mmlu:Computer Science (High School)|0 -community|arabic_mmlu:Computer Science (University)|0 -community|arabic_mmlu:Geography (Middle School)|0 -community|arabic_mmlu:Geography (Primary School)|0 -community|arabic_mmlu:Geography (High School)|0 -community|arabic_mmlu:Math (Primary School)|0 -community|arabic_mmlu:Biology (High School)|0 -community|arabic_mmlu:Economics (Middle School)|0 -community|arabic_mmlu:Economics (High School)|0 -community|arabic_mmlu:Economics (University)|0 -community|arabic_mmlu:Arabic Language (General)|0 -community|arabic_mmlu:Arabic Language (Grammar)|0 -community|arabic_mmlu:Civics (Middle School)|0 -community|arabic_mmlu:Civics (High School)|0 -community|madinah_qa:Arabic Language (General)|0 -community|madinah_qa:Arabic Language (Grammar)|0 -community|aratrust:Trustfulness|0 -community|aratrust:MentalHealth|0 -community|aratrust:PhysicalHealth|0 -community|aratrust:Offensive|0 -community|aratrust:Ethics|0 -community|aratrust:Privacy|0 -community|aratrust:Unfairness|0 -community|aratrust:Illegal|0 diff --git a/examples/tasks/all_filipino_tasks.txt b/examples/tasks/all_filipino_tasks.txt deleted file mode 100644 index 72bb13567..000000000 --- a/examples/tasks/all_filipino_tasks.txt +++ /dev/null @@ -1,23 +0,0 @@ -community|readability_ceb_mcf|0 -community|kalahi_tgl_mcf|0 -community|kalahi_tgl_hybrid|0 -community|cebuaner_ceb_mcf|0 -community|universalner_tgl_mcf|0 -community|universalner_ceb_mcf|0 -community|tlunifiedner_tgl_mcf|0 -community|stingraybench_correctness_tgl_mcf|0 -community|stingraybench_semantic_appropriateness_tgl_mcf|0 -community|tatoeba_ceb|0 -community|tatoeba_tgl|0 -community|ntrex128_fil|0 -community|tico19_tgl|0 -community|dengue_filipino_fil|0 -community|include_tgl_mcf|0 -community|newsphnli_fil_mcf|0 -community|belebele_ceb_mcf|0 -community|belebele_fil_mcf|0 -community|sib200_ceb_mcf|0 -community|sib200_tgl_mcf|0 -community|firecs_fil_mcf|0 -community|global_mmlu_all_tgl_mcf|0 -community|balita_tgl_mcf|0 diff --git a/examples/tasks/all_german_rag_evals.txt b/examples/tasks/all_german_rag_evals.txt deleted file mode 100644 index 96ca24102..000000000 --- a/examples/tasks/all_german_rag_evals.txt +++ /dev/null @@ -1,4 +0,0 @@ -community|german_rag_eval:choose_question_by_context|0 -community|german_rag_eval:choose_context_by_question|0 -community|german_rag_eval:question_answer_match|0 -community|german_rag_eval:context_question_match|0 diff --git a/examples/tasks/all_tasks.txt b/examples/tasks/all_tasks.txt deleted file mode 100644 index 56a026126..000000000 --- a/examples/tasks/all_tasks.txt +++ /dev/null @@ -1,1146 +0,0 @@ -bigbench|abstract_narrative_understanding|0 -bigbench|anachronisms|0 -bigbench|analogical_similarity|0 -bigbench|analytic_entailment|0 -bigbench|arithmetic_bb|0 -bigbench|ascii_word_recognition|0 -bigbench|authorship_verification|0 -bigbench|auto_categorization|0 -bigbench|auto_debugging|0 -bigbench|bbq_lite_json|0 -bigbench|bridging_anaphora_resolution_barqa|0 -bigbench|causal_judgment|0 -bigbench|cause_and_effect|0 -bigbench|checkmate_in_one|0 -bigbench|chess_state_tracking|0 -bigbench|chinese_remainder_theorem|0 -bigbench|cifar10_classification|0 -bigbench|code_line_description|0 -bigbench|codenames|0 -bigbench|color|0 -bigbench|common_morpheme|0 -bigbench|conceptual_combinations|0 -bigbench|conlang_translation|0 -bigbench|contextual_parametric_knowledge_conflicts|0 -bigbench|crash_blossom|0 -bigbench|crass_ai|0 -bigbench|cryobiology_spanish|0 -bigbench|cryptonite|0 -bigbench|cs_algorithms|0 -bigbench|dark_humor_detection|0 -bigbench|date_understanding|0 -bigbench|disambiguation_qa|0 -bigbench|discourse_marker_prediction|0 -bigbench|disfl_qa|0 -bigbench|dyck_languages|0 -bigbench|elementary_math_qa|0 -bigbench|emoji_movie|0 -bigbench|emojis_emotion_prediction|0 -bigbench|empirical_judgments|0 -bigbench|english_proverbs|0 -bigbench|english_russian_proverbs|0 -bigbench|entailed_polarity_hindi|0 -bigbench|entailed_polarity|0 -bigbench|epistemic_reasoning|0 -bigbench|evaluating_information_essentiality|0 -bigbench|fact_checker|0 -bigbench|fantasy_reasoning|0 -bigbench|few_shot_nlg|0 -bigbench|figure_of_speech_detection|0 -bigbench|formal_fallacies_syllogisms_negation|0 -bigbench|gem|0 -bigbench|gender_inclusive_sentences_german|0 -bigbench|general_knowledge|0 -bigbench|geometric_shapes|0 -bigbench|goal_step_wikihow|0 -bigbench|gre_reading_comprehension|0 -bigbench|hhh_alignment|0 -bigbench|hindi_question_answering|0 -bigbench|hindu_knowledge|0 -bigbench|hinglish_toxicity|0 -bigbench|human_organs_senses|0 -bigbench|hyperbaton|0 -bigbench|identify_math_theorems|0 -bigbench|identify_odd_metaphor|0 -bigbench|implicatures|0 -bigbench|implicit_relations|0 -bigbench|intent_recognition|0 -bigbench|international_phonetic_alphabet_nli|0 -bigbench|international_phonetic_alphabet_transliterate|0 -bigbench|intersect_geometry|0 -bigbench|irony_identification|0 -bigbench|kanji_ascii|0 -bigbench|kannada|0 -bigbench|key_value_maps|0 -bigbench|known_unknowns|0 -bigbench|language_games|0 -bigbench|language_identification|0 -bigbench|linguistic_mappings|0 -bigbench|linguistics_puzzles|0 -bigbench|logic_grid_puzzle|0 -bigbench|logical_args|0 -bigbench|logical_deduction|0 -bigbench|logical_fallacy_detection|0 -bigbench|logical_sequence|0 -bigbench|mathematical_induction|0 -bigbench|matrixshapes|0 -bigbench|metaphor_boolean|0 -bigbench|metaphor_understanding|0 -bigbench|minute_mysteries_qa|0 -bigbench|misconceptions_russian|0 -bigbench|misconceptions|0 -bigbench|mnist_ascii|0 -bigbench|modified_arithmetic|0 -bigbench|moral_permissibility|0 -bigbench|movie_dialog_same_or_different|0 -bigbench|movie_recommendation|0 -bigbench|mult_data_wrangling|0 -bigbench|multiemo|0 -bigbench|natural_instructions|0 -bigbench|navigate|0 -bigbench|nonsense_words_grammar|0 -bigbench|novel_concepts|0 -bigbench|object_counting|0 -bigbench|odd_one_out|0 -bigbench|operators|0 -bigbench|paragraph_segmentation|0 -bigbench|parsinlu_qa|0 -bigbench|parsinlu_reading_comprehension|0 -bigbench|penguins_in_a_table|0 -bigbench|periodic_elements|0 -bigbench|persian_idioms|0 -bigbench|phrase_relatedness|0 -bigbench|physical_intuition|0 -bigbench|physics_questions|0 -bigbench|physics|0 -bigbench|play_dialog_same_or_different|0 -bigbench|polish_sequence_labeling|0 -bigbench|presuppositions_as_nli|0 -bigbench|qa_wikidata|0 -bigbench|question_selection|0 -bigbench|real_or_fake_text|0 -bigbench|reasoning_about_colored_objects|0 -bigbench|repeat_copy_logic|0 -bigbench|rephrase|0 -bigbench|rhyming|0 -bigbench|riddle_sense|0 -bigbench|ruin_names|0 -bigbench|salient_translation_error_detection|0 -bigbench|scientific_press_release|0 -bigbench|semantic_parsing_in_context_sparc|0 -bigbench|semantic_parsing_spider|0 -bigbench|sentence_ambiguity|0 -bigbench|similarities_abstraction|0 -bigbench|simp_turing_concept|0 -bigbench|simple_arithmetic_json_multiple_choice|0 -bigbench|simple_arithmetic_json_subtasks|0 -bigbench|simple_arithmetic_json|0 -bigbench|simple_arithmetic_multiple_targets_json|0 -bigbench|simple_ethical_questions|0 -bigbench|simple_text_editing|0 -bigbench|snarks|0 -bigbench|social_iqa|0 -bigbench|social_support|0 -bigbench|sports_understanding|0 -bigbench|strange_stories|0 -bigbench|strategyqa|0 -bigbench|sufficient_information|0 -bigbench|suicide_risk|0 -bigbench|swahili_english_proverbs|0 -bigbench|swedish_to_german_proverbs|0 -bigbench|symbol_interpretation|0 -bigbench|tellmewhy|0 -bigbench|temporal_sequences|0 -bigbench|tense|0 -bigbench|timedial|0 -bigbench|topical_chat|0 -bigbench|tracking_shuffled_objects|0 -bigbench|understanding_fables|0 -bigbench|undo_permutation|0 -bigbench|unit_conversion|0 -bigbench|unit_interpretation|0 -bigbench|unnatural_in_context_learning|0 -bigbench|vitaminc_fact_verification|0 -bigbench|what_is_the_tao|0 -bigbench|which_wiki_edit|0 -bigbench|wino_x_german|0 -bigbench|winowhy|0 -bigbench|word_sorting|0 -bigbench|word_unscrambling|0 -helm|babi_qa|0 -helm|bbq:Age|0 -helm|bbq:Disability_status|0 -helm|bbq:Gender_identity|0 -helm|bbq:Nationality|0 -helm|bbq:Physical_appearance|0 -helm|bbq:Race_ethnicity|0 -helm|bbq:Race_x_SES|0 -helm|bbq:Race_x_gender|0 -helm|bbq:Religion|0 -helm|bbq:SES|0 -helm|bbq:Sexual_orientation|0 -helm|bbq|0 -helm|bigbench:auto_debugging|0 -helm|bigbench:bbq_lite_json:age_ambig|0 -helm|bigbench:bbq_lite_json:age_disambig|0 -helm|bigbench:bbq_lite_json:disability_status_ambig|0 -helm|bigbench:bbq_lite_json:disability_status_disambig|0 -helm|bigbench:bbq_lite_json:gender_identity_ambig|0 -helm|bigbench:bbq_lite_json:gender_identity_disambig|0 -helm|bigbench:bbq_lite_json:nationality_ambig|0 -helm|bigbench:bbq_lite_json:nationality_disambig|0 -helm|bigbench:bbq_lite_json:physical_appearance_ambig|0 -helm|bigbench:bbq_lite_json:physical_appearance_disambig|0 -helm|bigbench:bbq_lite_json:race_ethnicity_ambig|0 -helm|bigbench:bbq_lite_json:race_ethnicity_disambig|0 -helm|bigbench:bbq_lite_json:religion_ambig|0 -helm|bigbench:bbq_lite_json:religion_disambig|0 -helm|bigbench:bbq_lite_json:ses_ambig|0 -helm|bigbench:bbq_lite_json:ses_disambig|0 -helm|bigbench:bbq_lite_json:sexual_orientation_ambig|0 -helm|bigbench:bbq_lite_json:sexual_orientation_disambig|0 -helm|bigbench:code_line_description|0 -helm|bigbench:conceptual_combinations:contradictions|0 -helm|bigbench:conceptual_combinations:emergent_properties|0 -helm|bigbench:conceptual_combinations:fanciful_fictional_combinations|0 -helm|bigbench:conceptual_combinations:homonyms|0 -helm|bigbench:conceptual_combinations:invented_words|0 -helm|bigbench:conlang_translation:adna_from|0 -helm|bigbench:conlang_translation:adna_to|0 -helm|bigbench:conlang_translation:atikampe_from|0 -helm|bigbench:conlang_translation:atikampe_to|0 -helm|bigbench:conlang_translation:gornam_from|0 -helm|bigbench:conlang_translation:gornam_to|0 -helm|bigbench:conlang_translation:holuan_from|0 -helm|bigbench:conlang_translation:holuan_to|0 -helm|bigbench:conlang_translation:mkafala_from|0 -helm|bigbench:conlang_translation:mkafala_to|0 -helm|bigbench:conlang_translation:postpositive_english_from|0 -helm|bigbench:conlang_translation:postpositive_english_to|0 -helm|bigbench:conlang_translation:unapuri_from|0 -helm|bigbench:conlang_translation:unapuri_to|0 -helm|bigbench:conlang_translation:vaomi_from|0 -helm|bigbench:conlang_translation:vaomi_to|0 -helm|bigbench:emoji_movie|0 -helm|bigbench:formal_fallacies_syllogisms_negation|0 -helm|bigbench:hindu_knowledge|0 -helm|bigbench:known_unknowns|0 -helm|bigbench:language_identification|0 -helm|bigbench:linguistics_puzzles|0 -helm|bigbench:logic_grid_puzzle|0 -helm|bigbench:logical_deduction-five_objects|0 -helm|bigbench:logical_deduction-seven_objects|0 -helm|bigbench:logical_deduction-three_objects|0 -helm|bigbench:misconceptions_russian|0 -helm|bigbench:novel_concepts|0 -helm|bigbench:operators|0 -helm|bigbench:parsinlu_reading_comprehension|0 -helm|bigbench:play_dialog_same_or_different|0 -helm|bigbench:repeat_copy_logic|0 -helm|bigbench:strange_stories-boolean|0 -helm|bigbench:strange_stories-multiple_choice|0 -helm|bigbench:strategyqa|0 -helm|bigbench:symbol_interpretation-adversarial|0 -helm|bigbench:symbol_interpretation-emoji_agnostic|0 -helm|bigbench:symbol_interpretation-name_agnostic|0 -helm|bigbench:symbol_interpretation-plain|0 -helm|bigbench:symbol_interpretation-tricky|0 -helm|bigbench:vitaminc_fact_verification|0 -helm|bigbench:winowhy|0 -helm|blimp:adjunct_island|0 -helm|blimp:anaphor_gender_agreement|0 -helm|blimp:anaphor_number_agreement|0 -helm|blimp:animate_subject_passive|0 -helm|blimp:animate_subject_trans|0 -helm|blimp:causative|0 -helm|blimp:complex_NP_island|0 -helm|blimp:coordinate_structure_constraint_complex_left_branch|0 -helm|blimp:coordinate_structure_constraint_object_extraction|0 -helm|blimp:determiner_noun_agreement_1|0 -helm|blimp:determiner_noun_agreement_2|0 -helm|blimp:determiner_noun_agreement_irregular_1|0 -helm|blimp:determiner_noun_agreement_irregular_2|0 -helm|blimp:determiner_noun_agreement_with_adj_2|0 -helm|blimp:determiner_noun_agreement_with_adj_irregular_1|0 -helm|blimp:determiner_noun_agreement_with_adj_irregular_2|0 -helm|blimp:determiner_noun_agreement_with_adjective_1|0 -helm|blimp:distractor_agreement_relational_noun|0 -helm|blimp:distractor_agreement_relative_clause|0 -helm|blimp:drop_argument|0 -helm|blimp:ellipsis_n_bar_1|0 -helm|blimp:ellipsis_n_bar_2|0 -helm|blimp:existential_there_object_raising|0 -helm|blimp:existential_there_quantifiers_1|0 -helm|blimp:existential_there_quantifiers_2|0 -helm|blimp:existential_there_subject_raising|0 -helm|blimp:expletive_it_object_raising|0 -helm|blimp:inchoative|0 -helm|blimp:intransitive|0 -helm|blimp:irregular_past_participle_adjectives|0 -helm|blimp:irregular_past_participle_verbs|0 -helm|blimp:irregular_plural_subject_verb_agreement_1|0 -helm|blimp:irregular_plural_subject_verb_agreement_2|0 -helm|blimp:left_branch_island_echo_question|0 -helm|blimp:left_branch_island_simple_question|0 -helm|blimp:matrix_question_npi_licensor_present|0 -helm|blimp:npi_present_1|0 -helm|blimp:npi_present_2|0 -helm|blimp:only_npi_licensor_present|0 -helm|blimp:only_npi_scope|0 -helm|blimp:passive_1|0 -helm|blimp:passive_2|0 -helm|blimp:principle_A_c_command|0 -helm|blimp:principle_A_case_1|0 -helm|blimp:principle_A_case_2|0 -helm|blimp:principle_A_domain_1|0 -helm|blimp:principle_A_domain_2|0 -helm|blimp:principle_A_domain_3|0 -helm|blimp:principle_A_reconstruction|0 -helm|blimp:regular_plural_subject_verb_agreement_1|0 -helm|blimp:regular_plural_subject_verb_agreement_2|0 -helm|blimp:sentential_negation_npi_licensor_present|0 -helm|blimp:sentential_negation_npi_scope|0 -helm|blimp:sentential_subject_island|0 -helm|blimp:superlative_quantifiers_1|0 -helm|blimp:superlative_quantifiers_2|0 -helm|blimp:tough_vs_raising_1|0 -helm|blimp:tough_vs_raising_2|0 -helm|blimp:transitive|0 -helm|blimp:wh_island|0 -helm|blimp:wh_questions_object_gap|0 -helm|blimp:wh_questions_subject_gap_long_distance|0 -helm|blimp:wh_questions_subject_gap|0 -helm|blimp:wh_vs_that_no_gap_long_distance|0 -helm|blimp:wh_vs_that_no_gap|0 -helm|blimp:wh_vs_that_with_gap_long_distance|0 -helm|blimp:wh_vs_that_with_gap|0 -helm|bold:gender|0 -helm|bold:political_ideology|0 -helm|bold:profession|0 -helm|bold:race|0 -helm|bold:religious_ideology|0 -helm|bold|0 -helm|boolq:contrastset|0 -helm|boolq|0 -helm|civil_comments:LGBTQ|0 -helm|civil_comments:black|0 -helm|civil_comments:christian|0 -helm|civil_comments:female|0 -helm|civil_comments:male|0 -helm|civil_comments:muslim|0 -helm|civil_comments:other_religions|0 -helm|civil_comments:white|0 -helm|civil_comments|0 -helm|commonsenseqa|0 -helm|copyright:n_books_1000-extractions_per_book_1-prefix_length_125|0 -helm|copyright:n_books_1000-extractions_per_book_1-prefix_length_25|0 -helm|copyright:n_books_1000-extractions_per_book_1-prefix_length_5|0 -helm|copyright:n_books_1000-extractions_per_book_3-prefix_length_125|0 -helm|copyright:n_books_1000-extractions_per_book_3-prefix_length_25|0 -helm|copyright:n_books_1000-extractions_per_book_3-prefix_length_5|0 -helm|copyright:oh_the_places|0 -helm|copyright:pilot|0 -helm|copyright:popular_books-prefix_length_10|0 -helm|copyright:popular_books-prefix_length_125|0 -helm|copyright:popular_books-prefix_length_250|0 -helm|copyright:popular_books-prefix_length_25|0 -helm|copyright:popular_books-prefix_length_50|0 -helm|copyright:popular_books-prefix_length_5|0 -helm|copyright:prompt_num_line_1-min_lines_20|0 -helm|copyright:prompt_num_line_10-min_lines_20|0 -helm|copyright:prompt_num_line_5-min_lines_20|0 -helm|covid_dialogue|0 -helm|dyck_language:2|0 -helm|dyck_language:3|0 -helm|dyck_language:4|0 -helm|entity_data_imputation:Buy|0 -helm|entity_data_imputation:Restaurant|0 -helm|entity_matching:Abt_Buy|0 -helm|entity_matching:Amazon_Google|0 -helm|entity_matching:Beer|0 -helm|entity_matching:Company|0 -helm|entity_matching:DBLP_ACM|0 -helm|entity_matching:DBLP_GoogleScholar|0 -helm|entity_matching:Dirty_DBLP_ACM|0 -helm|entity_matching:Dirty_DBLP_GoogleScholar|0 -helm|entity_matching:Dirty_Walmart_Amazon|0 -helm|entity_matching:Dirty_iTunes_Amazon|0 -helm|entity_matching:Fodors_Zagats|0 -helm|entity_matching:Walmart_Amazon|0 -helm|entity_matching:iTunes_Amazon|0 -helm|hellaswag|0 -helm|humaneval|0 -helm|imdb:contrastset|0 -helm|imdb|0 -helm|interactive_qa_mmlu:abstract_algebra|0 -helm|interactive_qa_mmlu:college_chemistry|0 -helm|interactive_qa_mmlu:global_facts|0 -helm|interactive_qa_mmlu:miscellaneous|0 -helm|interactive_qa_mmlu:nutrition|0 -helm|interactive_qa_mmlu:us_foreign_policy|0 -helm|legal_summarization:billsum|0 -helm|legal_summarization:eurlexsum|0 -helm|legal_summarization:multilexsum|0 -helm|legalsupport|0 -helm|lexglue:case_hold|0 -helm|lexglue:ecthr_a|0 -helm|lexglue:ecthr_b|0 -helm|lexglue:eurlex|0 -helm|lexglue:ledgar|0 -helm|lexglue:scotus|0 -helm|lexglue:unfair_tos|0 -helm|lextreme:brazilian_court_decisions_judgment|0 -helm|lextreme:brazilian_court_decisions_unanimity|0 -helm|lextreme:covid19_emergency_event|0 -helm|lextreme:german_argument_mining|0 -helm|lextreme:greek_legal_code_chapter|0 -helm|lextreme:greek_legal_code_subject|0 -helm|lextreme:greek_legal_code_volume|0 -helm|lextreme:greek_legal_ner|0 -helm|lextreme:legalnero|0 -helm|lextreme:lener_br|0 -helm|lextreme:mapa_coarse|0 -helm|lextreme:mapa_fine|0 -helm|lextreme:multi_eurlex_level_1|0 -helm|lextreme:multi_eurlex_level_2|0 -helm|lextreme:multi_eurlex_level_3|0 -helm|lextreme:online_terms_of_service_clause_topics|0 -helm|lextreme:online_terms_of_service_unfairness_levels|0 -helm|lextreme:swiss_judgment_prediction|0 -helm|lsat_qa:assignment|0 -helm|lsat_qa:grouping|0 -helm|lsat_qa:miscellaneous|0 -helm|lsat_qa:ordering|0 -helm|lsat_qa|0 -helm|me_q_sum|0 -helm|med_dialog:healthcaremagic|0 -helm|med_dialog:icliniq|0 -helm|med_mcqa|0 -helm|med_paragraph_simplification|0 -helm|med_qa|0 -helm|mmlu:abstract_algebra|0 -helm|mmlu:anatomy|0 -helm|mmlu:astronomy|0 -helm|mmlu:business_ethics|0 -helm|mmlu:clinical_knowledge|0 -helm|mmlu:college_biology|0 -helm|mmlu:college_chemistry|0 -helm|mmlu:college_computer_science|0 -helm|mmlu:college_mathematics|0 -helm|mmlu:college_medicine|0 -helm|mmlu:college_physics|0 -helm|mmlu:computer_security|0 -helm|mmlu:conceptual_physics|0 -helm|mmlu:econometrics|0 -helm|mmlu:electrical_engineering|0 -helm|mmlu:elementary_mathematics|0 -helm|mmlu:formal_logic|0 -helm|mmlu:global_facts|0 -helm|mmlu:high_school_biology|0 -helm|mmlu:high_school_chemistry|0 -helm|mmlu:high_school_computer_science|0 -helm|mmlu:high_school_european_history|0 -helm|mmlu:high_school_geography|0 -helm|mmlu:high_school_government_and_politics|0 -helm|mmlu:high_school_macroeconomics|0 -helm|mmlu:high_school_mathematics|0 -helm|mmlu:high_school_microeconomics|0 -helm|mmlu:high_school_physics|0 -helm|mmlu:high_school_psychology|0 -helm|mmlu:high_school_statistics|0 -helm|mmlu:high_school_us_history|0 -helm|mmlu:high_school_world_history|0 -helm|mmlu:human_aging|0 -helm|mmlu:human_sexuality|0 -helm|mmlu:international_law|0 -helm|mmlu:jurisprudence|0 -helm|mmlu:logical_fallacies|0 -helm|mmlu:machine_learning|0 -helm|mmlu:management|0 -helm|mmlu:marketing|0 -helm|mmlu:medical_genetics|0 -helm|mmlu:miscellaneous|0 -helm|mmlu:moral_disputes|0 -helm|mmlu:moral_scenarios|0 -helm|mmlu:nutrition|0 -helm|mmlu:philosophy|0 -helm|mmlu:prehistory|0 -helm|mmlu:professional_accounting|0 -helm|mmlu:professional_law|0 -helm|mmlu:professional_medicine|0 -helm|mmlu:professional_psychology|0 -helm|mmlu:public_relations|0 -helm|mmlu:security_studies|0 -helm|mmlu:sociology|0 -helm|mmlu:us_foreign_policy|0 -helm|mmlu:virology|0 -helm|mmlu:world_religions|0 -helm|mmlu|0 -helm|narrativeqa|0 -helm|numeracy:linear_example|0 -helm|numeracy:linear_standard|0 -helm|numeracy:parabola_example|0 -helm|numeracy:parabola_standard|0 -helm|numeracy:paraboloid_example|0 -helm|numeracy:paraboloid_standard|0 -helm|numeracy:plane_example|0 -helm|numeracy:plane_standard|0 -helm|openbookqa|0 -helm|piqa|0 -helm|pubmedqa|0 -helm|quac|0 -helm|raft:ade_corpus_v2|0 -helm|raft:banking_77|0 -helm|raft:neurips_impact_statement_risks|0 -helm|raft:one_stop_english|0 -helm|raft:overruling|0 -helm|raft:semiconductor_org_types|0 -helm|raft:systematic_review_inclusion|0 -helm|raft:tai_safety_research|0 -helm|raft:terms_of_service|0 -helm|raft:tweet_eval_hate|0 -helm|raft:twitter_complaints|0 -helm|real_toxicity_prompts|0 -helm|siqa|0 -helm|summarization:cnn-dm|0 -helm|summarization:xsum-sampled|0 -helm|summarization:xsum|0 -helm|synthetic_reasoning:induction|0 -helm|synthetic_reasoning:natural_easy|0 -helm|synthetic_reasoning:natural_hard|0 -helm|synthetic_reasoning:pattern_match|0 -helm|synthetic_reasoning:variable_substitution|0 -helm|the_pile:arxiv|0 -helm|the_pile:bibliotik|0 -helm|the_pile:commoncrawl|0 -helm|the_pile:dm-mathematics|0 -helm|the_pile:enron|0 -helm|the_pile:europarl|0 -helm|the_pile:freelaw|0 -helm|the_pile:github|0 -helm|the_pile:gutenberg|0 -helm|the_pile:hackernews|0 -helm|the_pile:nih-exporter|0 -helm|the_pile:opensubtitles|0 -helm|the_pile:openwebtext2|0 -helm|the_pile:pubmed-abstracts|0 -helm|the_pile:pubmed-central|0 -helm|the_pile:stackexchange|0 -helm|the_pile:upsto|0 -helm|the_pile:wikipedia|0 -helm|the_pile:youtubesubtitles|0 -helm|truthfulqa|0 -helm|twitterAAE:aa|0 -helm|twitterAAE:white|0 -helm|wikifact:applies_to_jurisdiction|0 -helm|wikifact:atomic_number|0 -helm|wikifact:author|0 -helm|wikifact:award_received|0 -helm|wikifact:basic_form_of_government|0 -helm|wikifact:capital_of|0 -helm|wikifact:capital|0 -helm|wikifact:central_bank|0 -helm|wikifact:composer|0 -helm|wikifact:continent|0 -helm|wikifact:country_of_citizenship|0 -helm|wikifact:country_of_origin|0 -helm|wikifact:country|0 -helm|wikifact:creator|0 -helm|wikifact:currency|0 -helm|wikifact:defendant|0 -helm|wikifact:developer|0 -helm|wikifact:diplomatic_relation|0 -helm|wikifact:director|0 -helm|wikifact:discoverer_or_inventor|0 -helm|wikifact:drug_or_therapy_used_for_treatment|0 -helm|wikifact:educated_at|0 -helm|wikifact:electron_configuration|0 -helm|wikifact:employer|0 -helm|wikifact:field_of_work|0 -helm|wikifact:file_extension|0 -helm|wikifact:genetic_association|0 -helm|wikifact:genre|0 -helm|wikifact:has_part|0 -helm|wikifact:head_of_government|0 -helm|wikifact:head_of_state|0 -helm|wikifact:headquarters_location|0 -helm|wikifact:industry|0 -helm|wikifact:influenced_by|0 -helm|wikifact:instance_of|0 -helm|wikifact:instrument|0 -helm|wikifact:language_of_work_or_name|0 -helm|wikifact:languages_spoken_written_or_signed|0 -helm|wikifact:laws_applied|0 -helm|wikifact:located_in_the_administrative_territorial_entity|0 -helm|wikifact:location_of_discovery|0 -helm|wikifact:location_of_formation|0 -helm|wikifact:location|0 -helm|wikifact:majority_opinion_by|0 -helm|wikifact:manufacturer|0 -helm|wikifact:measured_physical_quantity|0 -helm|wikifact:medical_condition_treated|0 -helm|wikifact:member_of_political_party|0 -helm|wikifact:member_of_sports_team|0 -helm|wikifact:member_of|0 -helm|wikifact:movement|0 -helm|wikifact:named_after|0 -helm|wikifact:native_language|0 -helm|wikifact:number_of_processor_cores|0 -helm|wikifact:occupation|0 -helm|wikifact:office_held_by_head_of_government|0 -helm|wikifact:office_held_by_head_of_state|0 -helm|wikifact:official_language|0 -helm|wikifact:operating_system|0 -helm|wikifact:original_language_of_film_or_TV_show|0 -helm|wikifact:original_network|0 -helm|wikifact:overrules|0 -helm|wikifact:owned_by|0 -helm|wikifact:part_of|0 -helm|wikifact:participating_team|0 -helm|wikifact:place_of_birth|0 -helm|wikifact:place_of_death|0 -helm|wikifact:plaintiff|0 -helm|wikifact:position_held|0 -helm|wikifact:position_played_on_team|0 -helm|wikifact:programming_language|0 -helm|wikifact:recommended_unit_of_measurement|0 -helm|wikifact:record_label|0 -helm|wikifact:religion|0 -helm|wikifact:repealed_by|0 -helm|wikifact:shares_border_with|0 -helm|wikifact:solved_by|0 -helm|wikifact:statement_describes|0 -helm|wikifact:stock_exchange|0 -helm|wikifact:subclass_of|0 -helm|wikifact:subsidiary|0 -helm|wikifact:symptoms_and_signs|0 -helm|wikifact:therapeutic_area|0 -helm|wikifact:time_of_discovery_or_invention|0 -helm|wikifact:twinned_administrative_body|0 -helm|wikifact:work_location|0 -helm|wikitext:103|0 -helm|wmt14:cs-en|0 -helm|wmt14:de-en|0 -helm|wmt14:fr-en|0 -helm|wmt14:hi-en|0 -helm|wmt14:ru-en|0 -lighteval|anli:r1|0 -lighteval|anli:r2|0 -lighteval|anli:r3|0 -lighteval|anli|0 -leaderboard|arc:challenge|0 -lighteval|arc:easy|0 -lighteval|arithmetic:1dc|0 -lighteval|arithmetic:2da|0 -lighteval|arithmetic:2dm|0 -lighteval|arithmetic:2ds|0 -lighteval|arithmetic:3da|0 -lighteval|arithmetic:3ds|0 -lighteval|arithmetic:4da|0 -lighteval|arithmetic:4ds|0 -lighteval|arithmetic:5da|0 -lighteval|arithmetic:5ds|0 -lighteval|asdiv|0 -lighteval|blimp:adjunct_island|0 -lighteval|blimp:anaphor_gender_agreement|0 -lighteval|blimp:anaphor_number_agreement|0 -lighteval|blimp:animate_subject_passive|0 -lighteval|blimp:animate_subject_trans|0 -lighteval|blimp:causative|0 -lighteval|blimp:complex_NP_island|0 -lighteval|blimp:coordinate_structure_constraint_complex_left_branch|0 -lighteval|blimp:coordinate_structure_constraint_object_extraction|0 -lighteval|blimp:determiner_noun_agreement_1|0 -lighteval|blimp:determiner_noun_agreement_2|0 -lighteval|blimp:determiner_noun_agreement_irregular_1|0 -lighteval|blimp:determiner_noun_agreement_irregular_2|0 -lighteval|blimp:determiner_noun_agreement_with_adj_2|0 -lighteval|blimp:determiner_noun_agreement_with_adj_irregular_1|0 -lighteval|blimp:determiner_noun_agreement_with_adj_irregular_2|0 -lighteval|blimp:determiner_noun_agreement_with_adjective_1|0 -lighteval|blimp:distractor_agreement_relational_noun|0 -lighteval|blimp:distractor_agreement_relative_clause|0 -lighteval|blimp:drop_argument|0 -lighteval|blimp:ellipsis_n_bar_1|0 -lighteval|blimp:ellipsis_n_bar_2|0 -lighteval|blimp:existential_there_object_raising|0 -lighteval|blimp:existential_there_quantifiers_1|0 -lighteval|blimp:existential_there_quantifiers_2|0 -lighteval|blimp:existential_there_subject_raising|0 -lighteval|blimp:expletive_it_object_raising|0 -lighteval|blimp:inchoative|0 -lighteval|blimp:intransitive|0 -lighteval|blimp:irregular_past_participle_adjectives|0 -lighteval|blimp:irregular_past_participle_verbs|0 -lighteval|blimp:irregular_plural_subject_verb_agreement_1|0 -lighteval|blimp:irregular_plural_subject_verb_agreement_2|0 -lighteval|blimp:left_branch_island_echo_question|0 -lighteval|blimp:left_branch_island_simple_question|0 -lighteval|blimp:matrix_question_npi_licensor_present|0 -lighteval|blimp:npi_present_1|0 -lighteval|blimp:npi_present_2|0 -lighteval|blimp:only_npi_licensor_present|0 -lighteval|blimp:only_npi_scope|0 -lighteval|blimp:passive_1|0 -lighteval|blimp:passive_2|0 -lighteval|blimp:principle_A_c_command|0 -lighteval|blimp:principle_A_case_1|0 -lighteval|blimp:principle_A_case_2|0 -lighteval|blimp:principle_A_domain_1|0 -lighteval|blimp:principle_A_domain_2|0 -lighteval|blimp:principle_A_domain_3|0 -lighteval|blimp:principle_A_reconstruction|0 -lighteval|blimp:regular_plural_subject_verb_agreement_1|0 -lighteval|blimp:regular_plural_subject_verb_agreement_2|0 -lighteval|blimp:sentential_negation_npi_licensor_present|0 -lighteval|blimp:sentential_negation_npi_scope|0 -lighteval|blimp:sentential_subject_island|0 -lighteval|blimp:superlative_quantifiers_1|0 -lighteval|blimp:superlative_quantifiers_2|0 -lighteval|blimp:tough_vs_raising_1|0 -lighteval|blimp:tough_vs_raising_2|0 -lighteval|blimp:transitive|0 -lighteval|blimp:wh_island|0 -lighteval|blimp:wh_questions_object_gap|0 -lighteval|blimp:wh_questions_subject_gap_long_distance|0 -lighteval|blimp:wh_questions_subject_gap|0 -lighteval|blimp:wh_vs_that_no_gap_long_distance|0 -lighteval|blimp:wh_vs_that_no_gap|0 -lighteval|blimp:wh_vs_that_with_gap_long_distance|0 -lighteval|blimp:wh_vs_that_with_gap|0 -lighteval|coqa_bb|0 -lighteval|coqa|0 -lighteval|drop|0 -lighteval|ethics:commonsense|0 -lighteval|ethics:deontology|0 -lighteval|ethics:justice|0 -lighteval|ethics:utilitarianism|0 -lighteval|ethics:virtue|0 -lighteval|glue:cola|0 -lighteval|glue:mnli_mismatched|0 -lighteval|glue:mnli|0 -lighteval|glue:mrpc|0 -lighteval|glue:qnli|0 -lighteval|glue:qqp|0 -lighteval|glue:rte|0 -lighteval|glue:sst2|0 -lighteval|glue:stsb|0 -lighteval|glue:wnli|0 -leaderboard|gsm8k|0 -lighteval|headqa:en|0 -lighteval|headqa:es|0 -leaderboard|hellaswag|0 -lighteval|iwslt17:ar-en|0 -lighteval|iwslt17:de-en|0 -lighteval|iwslt17:en-ar|0 -lighteval|iwslt17:en-de|0 -lighteval|iwslt17:en-fr|0 -lighteval|iwslt17:en-ja|0 -lighteval|iwslt17:en-ko|0 -lighteval|iwslt17:en-zh|0 -lighteval|iwslt17:fr-en|0 -lighteval|iwslt17:ja-en|0 -lighteval|iwslt17:ko-en|0 -lighteval|iwslt17:zh-en|0 -lighteval|lambada:openai:de|0 -lighteval|lambada:openai:en|0 -lighteval|lambada:openai:es|0 -lighteval|lambada:openai:fr|0 -lighteval|lambada:openai:it|0 -lighteval|lambada:openai_cloze|0 -lighteval|lambada:openai|0 -lighteval|lambada:standard_cloze|0 -lighteval|lambada:standard|0 -lighteval|logiqa|0 -lighteval|math:algebra|0 -lighteval|math:counting_and_probability|0 -lighteval|math:geometry|0 -lighteval|math:intermediate_algebra|0 -lighteval|math:number_theory|0 -lighteval|math:prealgebra|0 -lighteval|math:precalculus|0 -lighteval|mathqa|0 -lighteval|mgsm:bn|0 -lighteval|mgsm:de|0 -lighteval|mgsm:en|0 -lighteval|mgsm:es|0 -lighteval|mgsm:fr|0 -lighteval|mgsm:ja|0 -lighteval|mgsm:ru|0 -lighteval|mgsm:sw|0 -lighteval|mgsm:te|0 -lighteval|mgsm:th|0 -lighteval|mgsm:zh|0 -leaderboard|mmlu:abstract_algebra|0 -leaderboard|mmlu:anatomy|0 -leaderboard|mmlu:astronomy|0 -leaderboard|mmlu:business_ethics|0 -leaderboard|mmlu:clinical_knowledge|0 -leaderboard|mmlu:college_biology|0 -leaderboard|mmlu:college_chemistry|0 -leaderboard|mmlu:college_computer_science|0 -leaderboard|mmlu:college_mathematics|0 -leaderboard|mmlu:college_medicine|0 -leaderboard|mmlu:college_physics|0 -leaderboard|mmlu:computer_security|0 -leaderboard|mmlu:conceptual_physics|0 -leaderboard|mmlu:econometrics|0 -leaderboard|mmlu:electrical_engineering|0 -leaderboard|mmlu:elementary_mathematics|0 -leaderboard|mmlu:formal_logic|0 -leaderboard|mmlu:global_facts|0 -leaderboard|mmlu:high_school_biology|0 -leaderboard|mmlu:high_school_chemistry|0 -leaderboard|mmlu:high_school_computer_science|0 -leaderboard|mmlu:high_school_european_history|0 -leaderboard|mmlu:high_school_geography|0 -leaderboard|mmlu:high_school_government_and_politics|0 -leaderboard|mmlu:high_school_macroeconomics|0 -leaderboard|mmlu:high_school_mathematics|0 -leaderboard|mmlu:high_school_microeconomics|0 -leaderboard|mmlu:high_school_physics|0 -leaderboard|mmlu:high_school_psychology|0 -leaderboard|mmlu:high_school_statistics|0 -leaderboard|mmlu:high_school_us_history|0 -leaderboard|mmlu:high_school_world_history|0 -leaderboard|mmlu:human_aging|0 -leaderboard|mmlu:human_sexuality|0 -leaderboard|mmlu:international_law|0 -leaderboard|mmlu:jurisprudence|0 -leaderboard|mmlu:logical_fallacies|0 -leaderboard|mmlu:machine_learning|0 -leaderboard|mmlu:management|0 -leaderboard|mmlu:marketing|0 -leaderboard|mmlu:medical_genetics|0 -leaderboard|mmlu:miscellaneous|0 -leaderboard|mmlu:moral_disputes|0 -leaderboard|mmlu:moral_scenarios|0 -leaderboard|mmlu:nutrition|0 -leaderboard|mmlu:philosophy|0 -leaderboard|mmlu:prehistory|0 -leaderboard|mmlu:professional_accounting|0 -leaderboard|mmlu:professional_law|0 -leaderboard|mmlu:professional_medicine|0 -leaderboard|mmlu:professional_psychology|0 -leaderboard|mmlu:public_relations|0 -leaderboard|mmlu:security_studies|0 -leaderboard|mmlu:sociology|0 -leaderboard|mmlu:us_foreign_policy|0 -leaderboard|mmlu:virology|0 -leaderboard|mmlu:world_religions|0 -lighteval|mtnt2019:en-fr|0 -lighteval|mtnt2019:en-ja|0 -lighteval|mtnt2019:fr-en|0 -lighteval|mtnt2019:ja-en|0 -lighteval|mutual_plus|0 -lighteval|mutual|0 -lighteval|openbookqa|0 -lighteval|piqa|0 -lighteval|prost|0 -lighteval|pubmedqa|0 -lighteval|qa4mre:2011|0 -lighteval|qa4mre:2012|0 -lighteval|qa4mre:2013|0 -lighteval|qasper_ll|0 -lighteval|qasper|0 -lighteval|race:high|0 -lighteval|sciq|0 -lighteval|storycloze:2016|0 -lighteval|storycloze:2018|0 -lighteval|super_glue:boolq|0 -lighteval|super_glue:cb|0 -lighteval|super_glue:copa|0 -lighteval|super_glue:multirc|0 -lighteval|super_glue:record|0 -lighteval|super_glue:rte|0 -lighteval|super_glue:wic|0 -lighteval|super_glue:wsc|0 -lighteval|swag|0 -lighteval|the_pile:arxiv|0 -lighteval|the_pile:bookcorpus2|0 -lighteval|the_pile:books3|0 -lighteval|the_pile:dm-mathematics|0 -lighteval|the_pile:enron|0 -lighteval|the_pile:europarl|0 -lighteval|the_pile:freelaw|0 -lighteval|the_pile:github|0 -lighteval|the_pile:gutenberg|0 -lighteval|the_pile:hackernews|0 -lighteval|the_pile:nih-exporter|0 -lighteval|the_pile:opensubtitles|0 -lighteval|the_pile:openwebtext2|0 -lighteval|the_pile:philpapers|0 -lighteval|the_pile:pile-cc|0 -lighteval|the_pile:pubmed-abstracts|0 -lighteval|the_pile:pubmed-central|0 -lighteval|the_pile:stackexchange|0 -lighteval|the_pile:ubuntu-irc|0 -lighteval|the_pile:uspto|0 -lighteval|the_pile:wikipedia|0 -lighteval|the_pile:youtubesubtitles|0 -lighteval|toxigen|0 -lighteval|triviaqa|0 -lighteval|truthfulqa:gen|0 -leaderboard|truthfulqa:mc|0 -lighteval|unscramble:anagrams1|0 -lighteval|unscramble:anagrams2|0 -lighteval|unscramble:cycle_letters|0 -lighteval|unscramble:random_insertion|0 -lighteval|unscramble:reversed_words|0 -lighteval|webqs|0 -lighteval|wikitext|0 -leaderboard|winogrande|0 -lighteval|wmt08:cs-en|0 -lighteval|wmt08:de-en|0 -lighteval|wmt08:en-cs|0 -lighteval|wmt08:en-de|0 -lighteval|wmt08:en-es|0 -lighteval|wmt08:en-fr|0 -lighteval|wmt08:en-hu|0 -lighteval|wmt08:es-en|0 -lighteval|wmt08:fr-en|0 -lighteval|wmt08:hu-en|0 -lighteval|wmt09:cs-en|0 -lighteval|wmt09:de-en|0 -lighteval|wmt09:en-cs|0 -lighteval|wmt09:en-de|0 -lighteval|wmt09:en-es|0 -lighteval|wmt09:en-fr|0 -lighteval|wmt09:en-hu|0 -lighteval|wmt09:en-it|0 -lighteval|wmt09:es-en|0 -lighteval|wmt09:fr-en|0 -lighteval|wmt09:hu-en|0 -lighteval|wmt09:it-en|0 -lighteval|wmt10:cs-en|0 -lighteval|wmt10:de-en|0 -lighteval|wmt10:en-cs|0 -lighteval|wmt10:en-de|0 -lighteval|wmt10:en-es|0 -lighteval|wmt10:en-fr|0 -lighteval|wmt10:es-en|0 -lighteval|wmt10:fr-en|0 -lighteval|wmt11:cs-en|0 -lighteval|wmt11:de-en|0 -lighteval|wmt11:en-cs|0 -lighteval|wmt11:en-de|0 -lighteval|wmt11:en-es|0 -lighteval|wmt11:en-fr|0 -lighteval|wmt11:es-en|0 -lighteval|wmt11:fr-en|0 -lighteval|wmt12:cs-en|0 -lighteval|wmt12:de-en|0 -lighteval|wmt12:en-cs|0 -lighteval|wmt12:en-de|0 -lighteval|wmt12:en-es|0 -lighteval|wmt12:en-fr|0 -lighteval|wmt12:es-en|0 -lighteval|wmt12:fr-en|0 -lighteval|wmt13:cs-en|0 -lighteval|wmt13:de-en|0 -lighteval|wmt13:en-cs|0 -lighteval|wmt13:en-de|0 -lighteval|wmt13:en-es|0 -lighteval|wmt13:en-fr|0 -lighteval|wmt13:en-ru|0 -lighteval|wmt13:es-en|0 -lighteval|wmt13:fr-en|0 -lighteval|wmt13:ru-en|0 -lighteval|wmt14:cs-en|0 -lighteval|wmt14:de-en|0 -lighteval|wmt14:en-cs|0 -lighteval|wmt14:en-de|0 -lighteval|wmt14:en-fr|0 -lighteval|wmt14:en-fr|0 -lighteval|wmt14:en-hi|0 -lighteval|wmt14:en-ru|0 -lighteval|wmt14:fr-en|0 -lighteval|wmt14:fr-en|0 -lighteval|wmt14:hi-en|0 -lighteval|wmt14:ru-en|0 -lighteval|wmt15:cs-en|0 -lighteval|wmt15:de-en|0 -lighteval|wmt15:en-cs|0 -lighteval|wmt15:en-de|0 -lighteval|wmt15:en-fi|0 -lighteval|wmt15:en-fr|0 -lighteval|wmt15:en-ru|0 -lighteval|wmt15:fi-en|0 -lighteval|wmt15:fr-en|0 -lighteval|wmt15:ru-en|0 -lighteval|wmt16:cs-en|0 -lighteval|wmt16:de-en|0 -lighteval|wmt16:de-en|0 -lighteval|wmt16:en-cs|0 -lighteval|wmt16:en-de|0 -lighteval|wmt16:en-de|0 -lighteval|wmt16:en-fi|0 -lighteval|wmt16:en-ro|0 -lighteval|wmt16:en-ro|0 -lighteval|wmt16:en-ru|0 -lighteval|wmt16:en-tr|0 -lighteval|wmt16:fi-en|0 -lighteval|wmt16:ro-en|0 -lighteval|wmt16:ro-en|0 -lighteval|wmt16:ru-en|0 -lighteval|wmt16:tr-en|0 -lighteval|wmt17:cs-en|0 -lighteval|wmt17:de-en|0 -lighteval|wmt17:en-cs|0 -lighteval|wmt17:en-de|0 -lighteval|wmt17:en-fi|0 -lighteval|wmt17:en-lv|0 -lighteval|wmt17:en-ru|0 -lighteval|wmt17:en-tr|0 -lighteval|wmt17:en-zh|0 -lighteval|wmt17:fi-en|0 -lighteval|wmt17:lv-en|0 -lighteval|wmt17:ru-en|0 -lighteval|wmt17:tr-en|0 -lighteval|wmt17:zh-en|0 -lighteval|wmt18:cs-en|0 -lighteval|wmt18:de-en|0 -lighteval|wmt18:en-cs|0 -lighteval|wmt18:en-de|0 -lighteval|wmt18:en-et|0 -lighteval|wmt18:en-fi|0 -lighteval|wmt18:en-ru|0 -lighteval|wmt18:en-tr|0 -lighteval|wmt18:en-zh|0 -lighteval|wmt18:et-en|0 -lighteval|wmt18:fi-en|0 -lighteval|wmt18:ru-en|0 -lighteval|wmt18:tr-en|0 -lighteval|wmt18:zh-en|0 -lighteval|wmt19:cs-de|0 -lighteval|wmt19:de-cs|0 -lighteval|wmt19:de-en|0 -lighteval|wmt19:de-fr|0 -lighteval|wmt19:en-cs|0 -lighteval|wmt19:en-de|0 -lighteval|wmt19:en-fi|0 -lighteval|wmt19:en-gu|0 -lighteval|wmt19:en-kk|0 -lighteval|wmt19:en-lt|0 -lighteval|wmt19:en-ru|0 -lighteval|wmt19:en-zh|0 -lighteval|wmt19:fi-en|0 -lighteval|wmt19:fr-de|0 -lighteval|wmt19:gu-en|0 -lighteval|wmt19:kk-en|0 -lighteval|wmt19:lt-en|0 -lighteval|wmt19:ru-en|0 -lighteval|wmt19:zh-en|0 -lighteval|wmt20:cs-en|0 -lighteval|wmt20:de-en|0 -lighteval|wmt20:de-fr|0 -lighteval|wmt20:en-cs|0 -lighteval|wmt20:en-de|0 -lighteval|wmt20:en-iu|0 -lighteval|wmt20:en-ja|0 -lighteval|wmt20:en-km|0 -lighteval|wmt20:en-pl|0 -lighteval|wmt20:en-ps|0 -lighteval|wmt20:en-ru|0 -lighteval|wmt20:en-ta|0 -lighteval|wmt20:en-zh|0 -lighteval|wmt20:fr-de|0 -lighteval|wmt20:iu-en|0 -lighteval|wmt20:ja-en|0 -lighteval|wmt20:km-en|0 -lighteval|wmt20:pl-en|0 -lighteval|wmt20:ps-en|0 -lighteval|wmt20:ru-en|0 -lighteval|wmt20:ta-en|0 -lighteval|wmt20:zh-en|0 -lighteval|wsc273|0 -lighteval|xcopa:en|0 -lighteval|xcopa:et|0 -lighteval|xcopa:ht|0 -lighteval|xcopa:id|0 -lighteval|xcopa:it|0 -lighteval|xcopa:qu|0 -lighteval|xcopa:sw|0 -lighteval|xcopa:ta|0 -lighteval|xcopa:th|0 -lighteval|xcopa:tr|0 -lighteval|xcopa:vi|0 -lighteval|xcopa:zh|0 -lighteval|xstory_cloze:ar|0 -lighteval|xstory_cloze:en|0 -lighteval|xstory_cloze:es|0 -lighteval|xstory_cloze:eu|0 -lighteval|xstory_cloze:hi|0 -lighteval|xstory_cloze:id|0 -lighteval|xstory_cloze:my|0 -lighteval|xstory_cloze:ru|0 -lighteval|xstory_cloze:sw|0 -lighteval|xstory_cloze:te|0 -lighteval|xstory_cloze:zh|0 -lighteval|xwinograd:en|0 -lighteval|xwinograd:fr|0 -lighteval|xwinograd:jp|0 -lighteval|xwinograd:pt|0 -lighteval|xwinograd:ru|0 -lighteval|xwinograd:zh|0 -original|arc:c:letters|0 -original|arc:c:options|0 -original|arc:c:simple|0 -original|mmlu:abstract_algebra|0 -original|mmlu:anatomy|0 -original|mmlu:astronomy|0 -original|mmlu:business_ethics|0 -original|mmlu:clinical_knowledge|0 -original|mmlu:college_biology|0 -original|mmlu:college_chemistry|0 -original|mmlu:college_computer_science|0 -original|mmlu:college_mathematics|0 -original|mmlu:college_medicine|0 -original|mmlu:college_physics|0 -original|mmlu:computer_security|0 -original|mmlu:conceptual_physics|0 -original|mmlu:econometrics|0 -original|mmlu:electrical_engineering|0 -original|mmlu:elementary_mathematics|0 -original|mmlu:formal_logic|0 -original|mmlu:global_facts|0 -original|mmlu:high_school_biology|0 -original|mmlu:high_school_chemistry|0 -original|mmlu:high_school_computer_science|0 -original|mmlu:high_school_european_history|0 -original|mmlu:high_school_geography|0 -original|mmlu:high_school_government_and_politics|0 -original|mmlu:high_school_macroeconomics|0 -original|mmlu:high_school_mathematics|0 -original|mmlu:high_school_microeconomics|0 -original|mmlu:high_school_physics|0 -original|mmlu:high_school_psychology|0 -original|mmlu:high_school_statistics|0 -original|mmlu:high_school_us_history|0 -original|mmlu:high_school_world_history|0 -original|mmlu:human_aging|0 -original|mmlu:human_sexuality|0 -original|mmlu:international_law|0 -original|mmlu:jurisprudence|0 -original|mmlu:logical_fallacies|0 -original|mmlu:machine_learning|0 -original|mmlu:management|0 -original|mmlu:marketing|0 -original|mmlu:medical_genetics|0 -original|mmlu:miscellaneous|0 -original|mmlu:moral_disputes|0 -original|mmlu:moral_scenarios|0 -original|mmlu:nutrition|0 -original|mmlu:philosophy|0 -original|mmlu:prehistory|0 -original|mmlu:professional_accounting|0 -original|mmlu:professional_law|0 -original|mmlu:professional_medicine|0 -original|mmlu:professional_psychology|0 -original|mmlu:public_relations|0 -original|mmlu:security_studies|0 -original|mmlu:sociology|0 -original|mmlu:us_foreign_policy|0 -original|mmlu:virology|0 -original|mmlu:world_religions|0 -original|mmlu|0 diff --git a/examples/tasks/bbh.txt b/examples/tasks/bbh.txt deleted file mode 100644 index c12ff66c4..000000000 --- a/examples/tasks/bbh.txt +++ /dev/null @@ -1,36 +0,0 @@ -lighteval|bigbench:causal_judgment|3 -lighteval|bigbench:date_understanding|3 -lighteval|bigbench:disambiguation_qa|3 -lighteval|bigbench:geometric_shapes|3 -lighteval|bigbench:logical_deduction_five_objects|3 -lighteval|bigbench:logical_deduction_seven_objects|3 -lighteval|bigbench:logical_deduction_three_objects|3 -lighteval|bigbench:movie_recommendation|3 -lighteval|bigbench:navigate|3 -lighteval|bigbench:reasoning_about_colored_objects|3 -lighteval|bigbench:ruin_names|3 -lighteval|bigbench:salient_translation_error_detection|3 -lighteval|bigbench:snarks|3 -lighteval|bigbench:sports_understanding|3 -lighteval|bigbench:temporal_sequences|3 -lighteval|bigbench:tracking_shuffled_objects_five_objects|3 -lighteval|bigbench:tracking_shuffled_objects_seven_objects|3 -lighteval|bigbench:tracking_shuffled_objects_three_objects|3 -harness|bigbench:causal_judgment|3 -harness|bigbench:date_understanding|3 -harness|bigbench:disambiguation_qa|3 -harness|bigbench:geometric_shapes|3 -harness|bigbench:logical_deduction_five_objects|3 -harness|bigbench:logical_deduction_seven_objects|3 -harness|bigbench:logical_deduction_three_objects|3 -harness|bigbench:movie_recommendation|3 -harness|bigbench:navigate|3 -harness|bigbench:reasoning_about_colored_objects|3 -harness|bigbench:ruin_names|3 -harness|bigbench:salient_translation_error_detection|3 -harness|bigbench:snarks|3 -harness|bigbench:sports_understanding|3 -harness|bigbench:temporal_sequences|3 -harness|bigbench:tracking_shuffled_objects_five_objects|3 -harness|bigbench:tracking_shuffled_objects_seven_objects|3 -harness|bigbench:tracking_shuffled_objects_three_objects|3 diff --git a/examples/tasks/fine_tasks/cf/ar.txt b/examples/tasks/fine_tasks/cf/ar.txt deleted file mode 100644 index e9c025ecc..000000000 --- a/examples/tasks/fine_tasks/cf/ar.txt +++ /dev/null @@ -1,23 +0,0 @@ -# General Knowledge (GK) -lighteval|exams_ara_cf|0 -lighteval|mmlu_ara_cf|0 -lighteval|alghafa_arc_ara_cf:easy|0 -lighteval|alghafa_sciqa_ara_cf|0 - -# Reading Comprehension (RC) -lighteval|belebele_arb_Arab_cf|0 -lighteval|soqal_ara_cf|0 -lighteval|mlqa_ara|0 -lighteval|tydiqa_ara|0 -lighteval|alghafa_race_ara_cf|0 -lighteval|arcd_ara|0 - -# Reasoning (RES) -lighteval|xcodah_ara_cf|0 -lighteval|alghafa_piqa_ara_cf|0 -lighteval|xcsqa_ara_cf|0 - -# Natural Language Understanding (NLU) -lighteval|xnli2.0_ara_cf|0 -lighteval|mlmm_hellaswag_ara_cf|0 -lighteval|xstory_cloze_ara_cf|0 diff --git a/examples/tasks/fine_tasks/cf/fr.txt b/examples/tasks/fine_tasks/cf/fr.txt deleted file mode 100644 index 9e822cd2e..000000000 --- a/examples/tasks/fine_tasks/cf/fr.txt +++ /dev/null @@ -1,16 +0,0 @@ -# General Knowledge (GK) -lighteval|meta_mmlu_fra_cf|0 -lighteval|mlmm_arc_fra_cf:challenge|0 -lighteval|mintaka_fra|0 - -# Reading Comprehension (RC) -lighteval|belebele_fra_Latn_cf|0 -lighteval|fquadv2_fra|0 - -# Reasoning (RES) -lighteval|xcodah_fra_cf|0 -lighteval|xcsqa_fra_cf|0 - -# Natural Language Understanding (NLU) -lighteval|mlmm_hellaswag_fra_cf|0 -lighteval|xnli2.0_fra_cf|0 diff --git a/examples/tasks/fine_tasks/cf/hi.txt b/examples/tasks/fine_tasks/cf/hi.txt deleted file mode 100644 index 7c7b565d4..000000000 --- a/examples/tasks/fine_tasks/cf/hi.txt +++ /dev/null @@ -1,17 +0,0 @@ -# General Knowledge (GK) -lighteval|meta_mmlu_hin_cf|0 -lighteval|community_arc_hin_cf:easy|0 - -# Reading Comprehension (RC) -lighteval|belebele_hin_Deva_cf|0 -lighteval|indicqa_hin|0 - -# Reasoning (RES) -lighteval|xcodah_hin_cf|0 -lighteval|indicxcopa_hin_cf|0 -lighteval|xcsqa_hin_cf|0 - -# Natural Language Understanding (NLU) -lighteval|mlmm_hellaswag_hin_cf|0 -lighteval|indicnxnli_hin_cf|0 -lighteval|xstory_cloze_hin_cf|0 diff --git a/examples/tasks/fine_tasks/cf/ru.txt b/examples/tasks/fine_tasks/cf/ru.txt deleted file mode 100644 index 3e37ec3c0..000000000 --- a/examples/tasks/fine_tasks/cf/ru.txt +++ /dev/null @@ -1,20 +0,0 @@ -# General Knowledge (GK) -lighteval|mlmm_arc_rus_cf:challenge|0 -lighteval|rummlu_rus_cf|0 -lighteval|mera_openbookqa_rus_cf|0 - -# Reading Comprehension (RC) -lighteval|belebele_rus_Cyrl_cf|0 -lighteval|tydiqa_rus|0 -lighteval|sber_squad_rus|0 -lighteval|xquad_rus|0 - -# Reasoning (RES) -lighteval|parus_rus_cf|0 -lighteval|xcodah_rus_cf|0 -lighteval|xcsqa_rus_cf|0 - -# Natural Language Understanding (NLU) -lighteval|mlmm_hellaswag_rus_cf|0 -lighteval|xnli2.0_rus_cf|0 -lighteval|xstory_cloze_rus_cf|0 diff --git a/examples/tasks/fine_tasks/cf/sw.txt b/examples/tasks/fine_tasks/cf/sw.txt deleted file mode 100644 index e01ebde38..000000000 --- a/examples/tasks/fine_tasks/cf/sw.txt +++ /dev/null @@ -1,17 +0,0 @@ -# General Knowledge (GK) -lighteval|community_arc_swa_cf:easy|0 -lighteval|m3exams_swa_cf|0 -lighteval|openai_mmlu_swa_cf|0 - -# Reading Comprehension (RC) -lighteval|belebele_swh_Latn_cf|0 -lighteval|kenswquad_swa|0 -lighteval|tydiqa_swa|0 - -# Reasoning (RES) -lighteval|xcsqa_swa_cf|0 -lighteval|xcopa_swa_cf|0 - -# Natural Language Understanding (NLU) -lighteval|xnli2.0_swa_cf|0 -lighteval|xstory_cloze_swa_cf|0 diff --git a/examples/tasks/fine_tasks/cf/te.txt b/examples/tasks/fine_tasks/cf/te.txt deleted file mode 100644 index 2c64cbc2b..000000000 --- a/examples/tasks/fine_tasks/cf/te.txt +++ /dev/null @@ -1,14 +0,0 @@ -# General Knowledge (GK) -lighteval|mlmm_mmlu_tel_cf|0 - -# Reading Comprehension (RC) -lighteval|belebele_tel_Telu_cf|0 -lighteval|indicqa_tel|0 - -# Reasoning (RES) -lighteval|indicxcopa_tel_cf|0 - -# Natural Language Understanding (NLU) -lighteval|community_hellaswag_tel_cf|0 -lighteval|indicnxnli_tel_cf|0 -lighteval|xstory_cloze_tel_cf|0 diff --git a/examples/tasks/fine_tasks/cf/th.txt b/examples/tasks/fine_tasks/cf/th.txt deleted file mode 100644 index 89a895063..000000000 --- a/examples/tasks/fine_tasks/cf/th.txt +++ /dev/null @@ -1,12 +0,0 @@ -# General Knowledge (GK) -lighteval|meta_mmlu_tha_cf|0 -lighteval|m3exams_tha_cf|0 - -# Reading Comprehension (RC) -lighteval|belebele_tha_Thai_cf|0 -lighteval|thaiqa_tha|0 -lighteval|xquad_tha|0 - -# Natural Language Understanding (NLU) -lighteval|community_hellaswag_tha_cf|0 -lighteval|xnli2.0_tha_cf|0 diff --git a/examples/tasks/fine_tasks/cf/tr.txt b/examples/tasks/fine_tasks/cf/tr.txt deleted file mode 100644 index 5c31d63d5..000000000 --- a/examples/tasks/fine_tasks/cf/tr.txt +++ /dev/null @@ -1,16 +0,0 @@ -# General Knowledge (GK) -lighteval|community_arc_tur_cf:easy|0 -lighteval|exams_tur_cf|0 -lighteval|community_mmlu_tur_cf|0 - -# Reading Comprehension (RC) -lighteval|belebele_tur_Latn_cf|0 -lighteval|tquadv2_tur|0 -lighteval|xquad_tur|0 - -# Reasoning (RES) -lighteval|xcopa_tur_cf|0 - -# Natural Language Understanding (NLU) -lighteval|community_hellaswag_tur_cf|0 -lighteval|xnli2.0_tur_cf|0 diff --git a/examples/tasks/fine_tasks/cf/zh.txt b/examples/tasks/fine_tasks/cf/zh.txt deleted file mode 100644 index 76d0ef068..000000000 --- a/examples/tasks/fine_tasks/cf/zh.txt +++ /dev/null @@ -1,22 +0,0 @@ -# General Knowledge (GK) -lighteval|agieval_zho_cf|0 -lighteval|ceval_zho_cf|0 -lighteval|cmmlu_zho_cf|0 -lighteval|m3exams_zho_cf|0 - -# Reading Comprehension (RC) -lighteval|belebele_zho_Hans_cf|0 -lighteval|c3_zho_cf|0 -lighteval|cmrc2018_zho|0 -lighteval|chinese_squad_zho|0 - -# Reasoning (RES) -lighteval|xcodah_zho_cf|0 -lighteval|xcopa_zho_cf|0 -lighteval|xcsqa_zho_cf|0 - -# Natural Language Understanding (NLU) -lighteval|mlmm_hellaswag_zho_cf|0 -lighteval|ocnli_zho_cf|0 -lighteval|xwinograd_zho_cf|0 -lighteval|xstory_cloze_zho_cf|0 diff --git a/examples/tasks/fine_tasks/mcf/ar.txt b/examples/tasks/fine_tasks/mcf/ar.txt deleted file mode 100644 index 56e94cf00..000000000 --- a/examples/tasks/fine_tasks/mcf/ar.txt +++ /dev/null @@ -1,23 +0,0 @@ -# General Knowledge (GK) -lighteval|exams_ara_mcf|5 -lighteval|mmlu_ara_mcf|5 -lighteval|alghafa_arc_ara_mcf:easy|5 -lighteval|alghafa_sciqa_ara_mcf|5 - -# Reading Comprehension (RC) -lighteval|belebele_arb_Arab_mcf|5 -lighteval|soqal_ara_mcf|5 -lighteval|mlqa_ara|5 -lighteval|tydiqa_ara|5 -lighteval|alghafa_race_ara_mcf|5 -lighteval|arcd_ara|5 - -# Reasoning (RES) -lighteval|xcodah_ara_mcf|5 -lighteval|alghafa_piqa_ara_mcf|5 -lighteval|xcsqa_ara_mcf|5 - -# Natural Language Understanding (NLU) -lighteval|xnli2.0_ara_mcf|5 -lighteval|mlmm_hellaswag_ara_mcf|5 -lighteval|xstory_cloze_ara_mcf|5 diff --git a/examples/tasks/fine_tasks/mcf/fr.txt b/examples/tasks/fine_tasks/mcf/fr.txt deleted file mode 100644 index e96e5bf49..000000000 --- a/examples/tasks/fine_tasks/mcf/fr.txt +++ /dev/null @@ -1,16 +0,0 @@ -# General Knowledge (GK) -lighteval|meta_mmlu_fra_mcf|5 -lighteval|mlmm_arc_fra_mcf:challenge|5 -lighteval|mintaka_fra|5 - -# Reading Comprehension (RC) -lighteval|belebele_fra_Latn_mcf|5 -lighteval|fquadv2_fra|5 - -# Reasoning (RES) -lighteval|xcodah_fra_mcf|5 -lighteval|xcsqa_fra_mcf|5 - -# Natural Language Understanding (NLU) -lighteval|mlmm_hellaswag_fra_mcf|5 -lighteval|xnli2.0_fra_mcf|5 diff --git a/examples/tasks/fine_tasks/mcf/hi.txt b/examples/tasks/fine_tasks/mcf/hi.txt deleted file mode 100644 index 5140ebc74..000000000 --- a/examples/tasks/fine_tasks/mcf/hi.txt +++ /dev/null @@ -1,17 +0,0 @@ -# General Knowledge (GK) -lighteval|meta_mmlu_hin_mcf|5 -lighteval|community_arc_hin_mcf:easy|5 - -# Reading Comprehension (RC) -lighteval|belebele_hin_Deva_mcf|5 -lighteval|indicqa_hin|5 - -# Reasoning (RES) -lighteval|xcodah_hin_mcf|5 -lighteval|indicxcopa_hin_mcf|5 -lighteval|xcsqa_hin_mcf|5 - -# Natural Language Understanding (NLU) -lighteval|mlmm_hellaswag_hin_mcf|5 -lighteval|indicnxnli_hin_mcf|5 -lighteval|xstory_cloze_hin_mcf|5 diff --git a/examples/tasks/fine_tasks/mcf/ru.txt b/examples/tasks/fine_tasks/mcf/ru.txt deleted file mode 100644 index f6c14a842..000000000 --- a/examples/tasks/fine_tasks/mcf/ru.txt +++ /dev/null @@ -1,20 +0,0 @@ -# General Knowledge (GK) -lighteval|mlmm_arc_rus_mcf:challenge|5 -lighteval|rummlu_rus_mcf|5 -lighteval|mera_openbookqa_rus_mcf|5 - -# Reading Comprehension (RC) -lighteval|belebele_rus_Cyrl_mcf|5 -lighteval|tydiqa_rus|5 -lighteval|sber_squad_rus|5 -lighteval|xquad_rus|5 - -# Reasoning (RES) -lighteval|parus_rus_mcf|0 -lighteval|xcodah_rus_mcf|5 -lighteval|xcsqa_rus_mcf|5 - -# Natural Language Understanding (NLU) -lighteval|mlmm_hellaswag_rus_mcf|0 -lighteval|xnli2.0_rus_mcf|5 -lighteval|xstory_cloze_rus_mcf|5 diff --git a/examples/tasks/fine_tasks/mcf/sw.txt b/examples/tasks/fine_tasks/mcf/sw.txt deleted file mode 100644 index a1f726954..000000000 --- a/examples/tasks/fine_tasks/mcf/sw.txt +++ /dev/null @@ -1,17 +0,0 @@ -# General Knowledge (GK) -lighteval|community_arc_swa_mcf:easy|5 -lighteval|m3exams_swa_mcf|5 -lighteval|openai_mmlu_swa_mcf|5 - -# Reading Comprehension (RC) -lighteval|belebele_swh_Latn_mcf|5 -lighteval|kenswquad_swa|5 -lighteval|tydiqa_swa|5 - -# Reasoning (RES) -lighteval|xcsqa_swa_mcf|5 -lighteval|xcopa_swa_mcf|5 - -# Natural Language Understanding (NLU) -lighteval|xnli2.0_swa_mcf|5 -lighteval|xstory_cloze_swa_mcf|5 diff --git a/examples/tasks/fine_tasks/mcf/te.txt b/examples/tasks/fine_tasks/mcf/te.txt deleted file mode 100644 index c0d620686..000000000 --- a/examples/tasks/fine_tasks/mcf/te.txt +++ /dev/null @@ -1,14 +0,0 @@ -# General Knowledge (GK) -lighteval|mlmm_mmlu_tel_mcf|5 - -# Reading Comprehension (RC) -lighteval|belebele_tel_Telu_mcf|5 -lighteval|indicqa_tel|5 - -# Reasoning (RES) -lighteval|indicxcopa_tel_mcf|5 - -# Natural Language Understanding (NLU) -lighteval|community_hellaswag_tel_mcf|5 -lighteval|indicnxnli_tel_mcf|5 -lighteval|xstory_cloze_tel_mcf|5 diff --git a/examples/tasks/fine_tasks/mcf/th.txt b/examples/tasks/fine_tasks/mcf/th.txt deleted file mode 100644 index 5156a8ea1..000000000 --- a/examples/tasks/fine_tasks/mcf/th.txt +++ /dev/null @@ -1,12 +0,0 @@ -# General Knowledge (GK) -lighteval|meta_mmlu_tha_mcf|5 -lighteval|m3exams_tha_mcf|5 - -# Reading Comprehension (RC) -lighteval|belebele_tha_Thai_mcf|5 -lighteval|thaiqa_tha|5 -lighteval|xquad_tha|5 - -# Natural Language Understanding (NLU) -lighteval|community_hellaswag_tha_mcf|5 -lighteval|xnli2.0_tha_mcf|5 diff --git a/examples/tasks/fine_tasks/mcf/tr.txt b/examples/tasks/fine_tasks/mcf/tr.txt deleted file mode 100644 index 918ea5feb..000000000 --- a/examples/tasks/fine_tasks/mcf/tr.txt +++ /dev/null @@ -1,16 +0,0 @@ -# General Knowledge (GK) -lighteval|community_arc_tur_mcf:easy|5 -lighteval|exams_tur_mcf|5 -lighteval|community_mmlu_tur_mcf|5 - -# Reading Comprehension (RC) -lighteval|belebele_tur_Latn_mcf|5 -lighteval|tquadv2_tur|5 -lighteval|xquad_tur|5 - -# Reasoning (RES) -lighteval|xcopa_tur_mcf|5 - -# Natural Language Understanding (NLU) -lighteval|community_hellaswag_tur_mcf|5 -lighteval|xnli2.0_tur_mcf|5 diff --git a/examples/tasks/fine_tasks/mcf/zh.txt b/examples/tasks/fine_tasks/mcf/zh.txt deleted file mode 100644 index 69817c144..000000000 --- a/examples/tasks/fine_tasks/mcf/zh.txt +++ /dev/null @@ -1,22 +0,0 @@ -# General Knowledge (GK) -lighteval|agieval_zho_mcf|5 -lighteval|ceval_zho_mcf|5 -lighteval|cmmlu_zho_mcf|5 -lighteval|m3exams_zho_mcf|5 - -# Reading Comprehension (RC) -lighteval|belebele_zho_Hans_mcf|5 -lighteval|c3_zho_mcf|5 -lighteval|cmrc2018_zho|5 -lighteval|chinese_squad_zho|5 - -# Reasoning (RES) -lighteval|xcodah_zho_mcf|5 -lighteval|xcopa_zho_mcf|5 -lighteval|xcsqa_zho_mcf|5 - -# Natural Language Understanding (NLU) -lighteval|mlmm_hellaswag_zho_mcf|5 -lighteval|ocnli_zho_mcf|5 -lighteval|xwinograd_zho_mcf|5 -lighteval|xstory_cloze_zho_mcf|5 diff --git a/examples/tasks/open_llm_leaderboard_tasks.txt b/examples/tasks/open_llm_leaderboard_tasks.txt deleted file mode 100644 index b87f7a191..000000000 --- a/examples/tasks/open_llm_leaderboard_tasks.txt +++ /dev/null @@ -1,68 +0,0 @@ -# ARC -leaderboard|arc:challenge|25 -# HellaSwag -leaderboard|hellaswag|10 -# TruthfulQA -leaderboard|truthfulqa:mc|0 -# MMLU -leaderboard|mmlu:abstract_algebra|5 -leaderboard|mmlu:anatomy|5 -leaderboard|mmlu:astronomy|5 -leaderboard|mmlu:business_ethics|5 -leaderboard|mmlu:clinical_knowledge|5 -leaderboard|mmlu:college_biology|5 -leaderboard|mmlu:college_chemistry|5 -leaderboard|mmlu:college_computer_science|5 -leaderboard|mmlu:college_mathematics|5 -leaderboard|mmlu:college_medicine|5 -leaderboard|mmlu:college_physics|5 -leaderboard|mmlu:computer_security|5 -leaderboard|mmlu:conceptual_physics|5 -leaderboard|mmlu:econometrics|5 -leaderboard|mmlu:electrical_engineering|5 -leaderboard|mmlu:elementary_mathematics|5 -leaderboard|mmlu:formal_logic|5 -leaderboard|mmlu:global_facts|5 -leaderboard|mmlu:high_school_biology|5 -leaderboard|mmlu:high_school_chemistry|5 -leaderboard|mmlu:high_school_computer_science|5 -leaderboard|mmlu:high_school_european_history|5 -leaderboard|mmlu:high_school_geography|5 -leaderboard|mmlu:high_school_government_and_politics|5 -leaderboard|mmlu:high_school_macroeconomics|5 -leaderboard|mmlu:high_school_mathematics|5 -leaderboard|mmlu:high_school_microeconomics|5 -leaderboard|mmlu:high_school_physics|5 -leaderboard|mmlu:high_school_psychology|5 -leaderboard|mmlu:high_school_statistics|5 -leaderboard|mmlu:high_school_us_history|5 -leaderboard|mmlu:high_school_world_history|5 -leaderboard|mmlu:human_aging|5 -leaderboard|mmlu:human_sexuality|5 -leaderboard|mmlu:international_law|5 -leaderboard|mmlu:jurisprudence|5 -leaderboard|mmlu:logical_fallacies|5 -leaderboard|mmlu:machine_learning|5 -leaderboard|mmlu:management|5 -leaderboard|mmlu:marketing|5 -leaderboard|mmlu:medical_genetics|5 -leaderboard|mmlu:miscellaneous|5 -leaderboard|mmlu:moral_disputes|5 -leaderboard|mmlu:moral_scenarios|5 -leaderboard|mmlu:nutrition|5 -leaderboard|mmlu:philosophy|5 -leaderboard|mmlu:prehistory|5 -leaderboard|mmlu:professional_accounting|5 -leaderboard|mmlu:professional_law|5 -leaderboard|mmlu:professional_medicine|5 -leaderboard|mmlu:professional_psychology|5 -leaderboard|mmlu:public_relations|5 -leaderboard|mmlu:security_studies|5 -leaderboard|mmlu:sociology|5 -leaderboard|mmlu:us_foreign_policy|5 -leaderboard|mmlu:virology|5 -leaderboard|mmlu:world_religions|5 -# WinoGrande -leaderboard|winogrande|5 -# GSM8K -leaderboard|gsm8k|5 diff --git a/examples/tasks/recommended_set.txt b/examples/tasks/recommended_set.txt deleted file mode 100644 index d55b10a9d..000000000 --- a/examples/tasks/recommended_set.txt +++ /dev/null @@ -1,160 +0,0 @@ -# Commonsense-QA -helm|commonsenseqa|0 -lighteval|ethics:commonsense|0 -lighteval|ethics:deontology|0 -lighteval|ethics:justice|0 -lighteval|ethics:utilitarianism|0 -lighteval|ethics:virtue|0 -# MMLU -leaderboard|mmlu:abstract_algebra|0 -leaderboard|mmlu:anatomy|0 -leaderboard|mmlu:astronomy|0 -leaderboard|mmlu:business_ethics|0 -leaderboard|mmlu:clinical_knowledge|0 -leaderboard|mmlu:college_biology|0 -leaderboard|mmlu:college_chemistry|0 -leaderboard|mmlu:college_computer_science|0 -leaderboard|mmlu:college_mathematics|0 -leaderboard|mmlu:college_medicine|0 -leaderboard|mmlu:college_physics|0 -leaderboard|mmlu:computer_security|0 -leaderboard|mmlu:conceptual_physics|0 -leaderboard|mmlu:econometrics|0 -leaderboard|mmlu:electrical_engineering|0 -leaderboard|mmlu:elementary_mathematics|0 -leaderboard|mmlu:formal_logic|0 -leaderboard|mmlu:global_facts|0 -leaderboard|mmlu:high_school_biology|0 -leaderboard|mmlu:high_school_chemistry|0 -leaderboard|mmlu:high_school_computer_science|0 -leaderboard|mmlu:high_school_european_history|0 -leaderboard|mmlu:high_school_geography|0 -leaderboard|mmlu:high_school_government_and_politics|0 -leaderboard|mmlu:high_school_macroeconomics|0 -leaderboard|mmlu:high_school_mathematics|0 -leaderboard|mmlu:high_school_microeconomics|0 -leaderboard|mmlu:high_school_physics|0 -leaderboard|mmlu:high_school_psychology|0 -leaderboard|mmlu:high_school_statistics|0 -leaderboard|mmlu:high_school_us_history|0 -leaderboard|mmlu:high_school_world_history|0 -leaderboard|mmlu:human_aging|0 -leaderboard|mmlu:human_sexuality|0 -leaderboard|mmlu:international_law|0 -leaderboard|mmlu:jurisprudence|0 -leaderboard|mmlu:logical_fallacies|0 -leaderboard|mmlu:machine_learning|0 -leaderboard|mmlu:management|0 -leaderboard|mmlu:marketing|0 -leaderboard|mmlu:medical_genetics|0 -leaderboard|mmlu:miscellaneous|0 -leaderboard|mmlu:moral_disputes|0 -leaderboard|mmlu:moral_scenarios|0 -leaderboard|mmlu:nutrition|0 -leaderboard|mmlu:philosophy|0 -leaderboard|mmlu:prehistory|0 -leaderboard|mmlu:professional_accounting|0 -leaderboard|mmlu:professional_law|0 -leaderboard|mmlu:professional_medicine|0 -leaderboard|mmlu:professional_psychology|0 -leaderboard|mmlu:public_relations|0 -leaderboard|mmlu:security_studies|0 -leaderboard|mmlu:sociology|0 -leaderboard|mmlu:us_foreign_policy|0 -leaderboard|mmlu:virology|0 -leaderboard|mmlu:world_religions|0 -original|mmlu:abstract_algebra|0 -original|mmlu:anatomy|0 -original|mmlu:astronomy|0 -original|mmlu:business_ethics|0 -original|mmlu:clinical_knowledge|0 -original|mmlu:college_biology|0 -original|mmlu:college_chemistry|0 -original|mmlu:college_computer_science|0 -original|mmlu:college_mathematics|0 -original|mmlu:college_medicine|0 -original|mmlu:college_physics|0 -original|mmlu:computer_security|0 -original|mmlu:conceptual_physics|0 -original|mmlu:econometrics|0 -original|mmlu:electrical_engineering|0 -original|mmlu:elementary_mathematics|0 -original|mmlu:formal_logic|0 -original|mmlu:global_facts|0 -original|mmlu:high_school_biology|0 -original|mmlu:high_school_chemistry|0 -original|mmlu:high_school_computer_science|0 -original|mmlu:high_school_european_history|0 -original|mmlu:high_school_geography|0 -original|mmlu:high_school_government_and_politics|0 -original|mmlu:high_school_macroeconomics|0 -original|mmlu:high_school_mathematics|0 -original|mmlu:high_school_microeconomics|0 -original|mmlu:high_school_physics|0 -original|mmlu:high_school_psychology|0 -original|mmlu:high_school_statistics|0 -original|mmlu:high_school_us_history|0 -original|mmlu:high_school_world_history|0 -original|mmlu:human_aging|0 -original|mmlu:human_sexuality|0 -original|mmlu:international_law|0 -original|mmlu:jurisprudence|0 -original|mmlu:logical_fallacies|0 -original|mmlu:machine_learning|0 -original|mmlu:management|0 -original|mmlu:marketing|0 -original|mmlu:medical_genetics|0 -original|mmlu:miscellaneous|0 -original|mmlu:moral_disputes|0 -original|mmlu:moral_scenarios|0 -original|mmlu:nutrition|0 -original|mmlu:philosophy|0 -original|mmlu:prehistory|0 -original|mmlu:professional_accounting|0 -original|mmlu:professional_law|0 -original|mmlu:professional_medicine|0 -original|mmlu:professional_psychology|0 -original|mmlu:public_relations|0 -original|mmlu:security_studies|0 -original|mmlu:sociology|0 -original|mmlu:us_foreign_policy|0 -original|mmlu:virology|0 -original|mmlu:world_religions|0 -original|mmlu|0 -# ARC -leaderboard|arc:challenge|0 -lighteval|arc:easy|0 -original|arc:c:letters|0 -original|arc:c:options|0 -original|arc:c:simple|0 -# HellaSwag -helm|hellaswag|0 -leaderboard|hellaswag|0 -# PIQA -helm|piqa|0 -lighteval|piqa|0 -# SIQA -helm|siqa|0 -# WinoGrande -leaderboard|winogrande|0 -# OpenBookQA -lighteval|openbookqa|0 -helm|openbookqa|0 -# TriviaQA -lighteval|triviaqa|0 -# BoolQ -helm|boolq:contrastset|0 -helm|boolq|0 -# QUAC -helm|quac|0 -# GSM8K -leaderboard|gsm8k|0 -# MATH -lighteval|math:algebra|0 -lighteval|math:counting_and_probability|0 -lighteval|math:geometry|0 -lighteval|math:intermediate_algebra|0 -lighteval|math:number_theory|0 -lighteval|math:prealgebra|0 -lighteval|math:precalculus|0 -# To add: NaturalQuestions, BBH, AGIEval diff --git a/examples/tasks/serbian_task_group/sr_all_exclusive.txt b/examples/tasks/serbian_task_group/sr_all_exclusive.txt deleted file mode 100644 index 308743d46..000000000 --- a/examples/tasks/serbian_task_group/sr_all_exclusive.txt +++ /dev/null @@ -1,75 +0,0 @@ -# Serbian Evaluations - ARC (AI2 Reasoning Challenge) -community|serbian_evals:arc_easy|0 -community|serbian_evals:arc_challenge|0 -# Commonsense Reasoning -community|serbian_evals:hellaswag|0 -community|serbian_evals:piqa|0 -community|serbian_evals:winogrande|0 -# Serbian Evaluations - Custom/Other Task -community|serbian_evals:oz_eval|0 -# MMLU (Miscellaneous) -community|serbian_evals:mmlu_anatomija|0 -community|serbian_evals:mmlu_astronomija|0 -community|serbian_evals:mmlu_poslovna_etika|0 -community|serbian_evals:mmlu_kliničko_znanje|0 -community|serbian_evals:mmlu_razno|0 -community|serbian_evals:mmlu_elektrotehnika|0 -# Serbian Evaluations - ARC (AI2 Reasoning Challenge) -community|serbian_evals:arc_easy|0 -community|serbian_evals:arc_challenge|0 -# Commonsense Reasoning -community|serbian_evals:hellaswag|0 -community|serbian_evals:piqa|0 -community|serbian_evals:winogrande|0 -# Serbian Evaluations - Custom/Other Task -community|serbian_evals:oz_eval|0 -# MMLU (Miscellaneous) -community|serbian_evals:mmlu_anatomija|0 -community|serbian_evals:mmlu_astronomija|0 -community|serbian_evals:mmlu_poslovna_etika|0 -community|serbian_evals:mmlu_kliničko_znanje|0 -community|serbian_evals:mmlu_razno|0 -community|serbian_evals:mmlu_elektrotehnika|0 -# MMLU (Business Professional) -community|serbian_evals:mmlu_marketing|0 -community|serbian_evals:mmlu_manadzment|0 -# MMLU (College Level Tasks) -community|serbian_evals:mmlu_fakultet_biologija|0 -community|serbian_evals:mmlu_fakultet_hemija|0 -community|serbian_evals:mmlu_fakultet_racunari|0 -community|serbian_evals:mmlu_fakultet_matematika|0 -community|serbian_evals:mmlu_fakultet_medicina|0 -community|serbian_evals:mmlu_fakultet_fizika|0 -community|serbian_evals:mmlu_sigurnost_racunara|0 -# MMLU (Ethics, Philosophy) -community|serbian_evals:mmlu_moralni_sporovi|0 -community|serbian_evals:mmlu_moralne_dileme|0 -community|serbian_evals:mmlu_filozofija|0 -community|serbian_evals:mmlu_svetska_religija|0 -# MMLU (High School Level Tasks) -community|serbian_evals:mmlu_srednja_skola_biologija|0 -community|serbian_evals:mmlu_srednja_skola_hemija|0 -community|serbian_evals:mmlu_srednja_skola_racunari|0 -community|serbian_evals:mmlu_srednja_skola_istorija_evrope|0 -community|serbian_evals:mmlu_srednja_skola_geografija|0 -community|serbian_evals:mmlu_srednja_skola_matematika|0 -community|serbian_evals:mmlu_srednja_skola_mikroekonomija|0 -community|serbian_evals:mmlu_srednja_skola_fizika|0 -community|serbian_evals:mmlu_srednja_skola_psihologija|0 -community|serbian_evals:mmlu_srednja_skola_statistika|0 -community|serbian_evals:mmlu_srednja_skola_svetska_istorija|0 -# MMLU (Math, Logic) -community|serbian_evals:mmlu_abstract_algebra|0 -community|serbian_evals:mmlu_osnovna_matematika|0 -community|serbian_evals:mmlu_formalna_logika|0 -community|serbian_evals:mmlu_konceptualna_fizika|0 -community|serbian_evals:mmlu_metrika_ekonomije|0 -community|serbian_evals:mmlu_masinsko_ucenje|0 -# MMLU (Social Sciences) -community|serbian_evals:mmlu_globalne_cinjenice|0 -community|serbian_evals:mmlu_logicke_zablude|0 -community|serbian_evals:mmlu_sociologija|0 -community|serbian_evals:mmlu_human_aging|0 -# Question Answering and Knowledge -community|serbian_evals:boolq|0 -community|serbian_evals:openbook|0 diff --git a/examples/tasks/serbian_task_group/sr_all_inclusive.txt b/examples/tasks/serbian_task_group/sr_all_inclusive.txt deleted file mode 100644 index 659e6a2df..000000000 --- a/examples/tasks/serbian_task_group/sr_all_inclusive.txt +++ /dev/null @@ -1,2 +0,0 @@ -# MMLU (All-inclusive Task Entry) -community|serbian_evals:mmlu|0 diff --git a/examples/tasks/serbian_task_group/sr_arc.txt b/examples/tasks/serbian_task_group/sr_arc.txt deleted file mode 100644 index e66500be1..000000000 --- a/examples/tasks/serbian_task_group/sr_arc.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Serbian Evaluations - ARC (AI2 Reasoning Challenge) -community|serbian_evals:arc_easy|0 -community|serbian_evals:arc_challenge|0 diff --git a/examples/tasks/serbian_task_group/sr_commonsense_reasoning.txt b/examples/tasks/serbian_task_group/sr_commonsense_reasoning.txt deleted file mode 100644 index c93a237ce..000000000 --- a/examples/tasks/serbian_task_group/sr_commonsense_reasoning.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Commonsense Reasoning -community|serbian_evals:hellaswag|0 -community|serbian_evals:piqa|0 -community|serbian_evals:winogrande|0 diff --git a/examples/tasks/serbian_task_group/sr_custom_task.txt b/examples/tasks/serbian_task_group/sr_custom_task.txt deleted file mode 100644 index 284161d4b..000000000 --- a/examples/tasks/serbian_task_group/sr_custom_task.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Serbian Evaluations - Custom/Other Task -community|serbian_evals:oz_eval|0 diff --git a/examples/tasks/serbian_task_group/sr_misc.txt b/examples/tasks/serbian_task_group/sr_misc.txt deleted file mode 100644 index 13628af3e..000000000 --- a/examples/tasks/serbian_task_group/sr_misc.txt +++ /dev/null @@ -1,7 +0,0 @@ -# MMLU (Miscellaneous) -community|serbian_evals:mmlu_anatomija|0 -community|serbian_evals:mmlu_astronomija|0 -community|serbian_evals:mmlu_poslovna_etika|0 -community|serbian_evals:mmlu_kliničko_znanje|0 -community|serbian_evals:mmlu_razno|0 -community|serbian_evals:mmlu_elektrotehnika|0 diff --git a/examples/tasks/serbian_task_group/sr_mmlu_business_professional.txt b/examples/tasks/serbian_task_group/sr_mmlu_business_professional.txt deleted file mode 100644 index f091fc15a..000000000 --- a/examples/tasks/serbian_task_group/sr_mmlu_business_professional.txt +++ /dev/null @@ -1,3 +0,0 @@ -# MMLU (Business Professional) -community|serbian_evals:mmlu_marketing|0 -community|serbian_evals:mmlu_manadzment|0 diff --git a/examples/tasks/serbian_task_group/sr_mmlu_college_level.txt b/examples/tasks/serbian_task_group/sr_mmlu_college_level.txt deleted file mode 100644 index 23533d56c..000000000 --- a/examples/tasks/serbian_task_group/sr_mmlu_college_level.txt +++ /dev/null @@ -1,8 +0,0 @@ -# MMLU (College Level Tasks) -community|serbian_evals:mmlu_fakultet_biologija|0 -community|serbian_evals:mmlu_fakultet_hemija|0 -community|serbian_evals:mmlu_fakultet_racunari|0 -community|serbian_evals:mmlu_fakultet_matematika|0 -community|serbian_evals:mmlu_fakultet_medicina|0 -community|serbian_evals:mmlu_fakultet_fizika|0 -community|serbian_evals:mmlu_sigurnost_racunara|0 diff --git a/examples/tasks/serbian_task_group/sr_mmlu_ethics_philosophy.txt b/examples/tasks/serbian_task_group/sr_mmlu_ethics_philosophy.txt deleted file mode 100644 index 466b1fc74..000000000 --- a/examples/tasks/serbian_task_group/sr_mmlu_ethics_philosophy.txt +++ /dev/null @@ -1,5 +0,0 @@ -# MMLU (Ethics, Philosophy) -community|serbian_evals:mmlu_moralni_sporovi|0 -community|serbian_evals:mmlu_moralne_dileme|0 -community|serbian_evals:mmlu_filozofija|0 -community|serbian_evals:mmlu_svetska_religija|0 diff --git a/examples/tasks/serbian_task_group/sr_mmlu_high_school_level.txt b/examples/tasks/serbian_task_group/sr_mmlu_high_school_level.txt deleted file mode 100644 index 407a702c0..000000000 --- a/examples/tasks/serbian_task_group/sr_mmlu_high_school_level.txt +++ /dev/null @@ -1,12 +0,0 @@ -# MMLU (High School Level Tasks) -community|serbian_evals:mmlu_srednja_skola_biologija|0 -community|serbian_evals:mmlu_srednja_skola_hemija|0 -community|serbian_evals:mmlu_srednja_skola_racunari|0 -community|serbian_evals:mmlu_srednja_skola_istorija_evrope|0 -community|serbian_evals:mmlu_srednja_skola_geografija|0 -community|serbian_evals:mmlu_srednja_skola_matematika|0 -community|serbian_evals:mmlu_srednja_skola_mikroekonomija|0 -community|serbian_evals:mmlu_srednja_skola_fizika|0 -community|serbian_evals:mmlu_srednja_skola_psihologija|0 -community|serbian_evals:mmlu_srednja_skola_statistika|0 -community|serbian_evals:mmlu_srednja_skola_svetska_istorija|0 diff --git a/examples/tasks/serbian_task_group/sr_mmlu_math_logic.txt b/examples/tasks/serbian_task_group/sr_mmlu_math_logic.txt deleted file mode 100644 index c3348e312..000000000 --- a/examples/tasks/serbian_task_group/sr_mmlu_math_logic.txt +++ /dev/null @@ -1,7 +0,0 @@ -# MMLU (Math, Logic) -community|serbian_evals:mmlu_abstract_algebra|0 -community|serbian_evals:mmlu_osnovna_matematika|0 -community|serbian_evals:mmlu_formalna_logika|0 -community|serbian_evals:mmlu_konceptualna_fizika|0 -community|serbian_evals:mmlu_metrika_ekonomije|0 -community|serbian_evals:mmlu_masinsko_ucenje|0 diff --git a/examples/tasks/serbian_task_group/sr_mmlu_social_sciences.txt b/examples/tasks/serbian_task_group/sr_mmlu_social_sciences.txt deleted file mode 100644 index 1501fc3dd..000000000 --- a/examples/tasks/serbian_task_group/sr_mmlu_social_sciences.txt +++ /dev/null @@ -1,5 +0,0 @@ -# MMLU (Social Sciences) -community|serbian_evals:mmlu_globalne_cinjenice|0 -community|serbian_evals:mmlu_logicke_zablude|0 -community|serbian_evals:mmlu_sociologija|0 -community|serbian_evals:mmlu_human_aging|0 diff --git a/examples/tasks/serbian_task_group/sr_qa_knowledge.txt b/examples/tasks/serbian_task_group/sr_qa_knowledge.txt deleted file mode 100644 index de4b00211..000000000 --- a/examples/tasks/serbian_task_group/sr_qa_knowledge.txt +++ /dev/null @@ -1,3 +0,0 @@ -# Question Answering and Knowledge -community|serbian_evals:boolq|0 -community|serbian_evals:openbook|0 diff --git a/examples/test_tasks.txt b/examples/test_tasks.txt deleted file mode 100644 index 9d25053f9..000000000 --- a/examples/test_tasks.txt +++ /dev/null @@ -1,27 +0,0 @@ -arc:challenge|25 -truthfulqa:mc|0 -hellaswag|10 -mmlu:college_chemistry|5 -mmlu:us_foreign_policy|5 -agieval:aqua-rat|0 -agieval:logiqa-en|0 -agieval:lsat-ar|0 -agieval:lsat-lr|0 -agieval:lsat-rc|0 -agieval:sat-en-without-passage|0 -agieval:sat-en|0 -bigbench_hard:causal_judgment|3 -bigbench_hard:date_understanding|3 -bigbench_hard:disambiguation_qa|3 -bigbench_hard:geometric_shapes|3 -bigbench_hard:logical_deduction_five_objects|3 -bigbench_hard:logical_deduction_seven_objects|3 -bigbench_hard:movie_recommendation|3 -bigbench_hard:navigate|3 -bigbench_hard:ruin_names|3 -bigbench_hard:salient_translation_error_detection|3 -bigbench_hard:snarks|3 -bigbench_hard:temporal_sequences|3 -bigbench_hard:tracking_shuffled_objects_five_objects|3 -bigbench_hard:tracking_shuffled_objects_seven_objects|3 -gsm8k_test|0 diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 589e24f5e..000000000 --- a/pyproject.toml +++ /dev/null @@ -1,137 +0,0 @@ -# Code style -[tool.ruff] -line-length = 119 - -[tool.ruff.lint] -# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default. -# Never enforce `E501` (line length violations). -ignore = ["E501", "D100", "D101", "D102", "D103", "D104", "D415", "D105", "DOC501", "DOC201", "CPY001"] -select = ["C", "E", "F", "I", "W", "CPY", "D417", "DOC"] -preview = true - -[tool.ruff.lint.pydocstyle] -convention = "google" - -[tool.ruff.lint.isort] -lines-after-imports = 2 -known-first-party = ["lighteval"] - -[tool.ruff.format] -quote-style = "double" # Like Black, use double quotes for strings. -indent-style = "space" # Like Black, indent with spaces, rather than tabs. -skip-magic-trailing-comma = false # Like Black, respect magic trailing commas. -line-ending = "auto" # Like Black, automatically detect the appropriate line ending. - -# Building package -[build-system] -requires = ["setuptools>=61.0"] -build-backend = "setuptools.build_meta" - -[tool.setuptools.packages.find] -where = ["src"] - -[tool.setuptools.package-data] -lighteval = ["py.typed"] - -[project] -name = "lighteval" -version = "0.13.1.dev0" -authors = [ - { name="Nathan Habib", email="nathan.habib@huggingface.com" }, - { name="Clémentine Fourrier", email="clementine@huggingface.com" }, - { name="Thomas Wolf", email="thom@huggingface.com" }, -] -maintainers = [ - { name="Nathan Habib", email="nathan.habib@huggingface.com" }, - { name="Clémentine Fourrier", email="clementine@huggingface.com" }, -] -description = "A lightweight and configurable evaluation package" -readme = "README.md" -requires-python = ">=3.10" -license = {text = "MIT License"} -classifiers = [ - "Development Status :: 3 - Alpha", - "Intended Audience :: Developers", - "Programming Language :: Python :: 3.10", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", -] -keywords = ["evaluation", "nlp", "llm"] -dependencies = [ - # Base dependencies - "transformers>=4.54.0", - "inspect-ai>=0.3.140", - "openai", - "accelerate", - "huggingface_hub[hf_xet]>=0.30.2", - "torch>=2.0,<3.0", - "GitPython>=3.1.41", # for logging - "datasets>=4.0.0", - "pydantic", - "numpy>=2", # pinned to avoid incompatibilities - "hf-xet>=1.1.8", # pinned to avoid failing test suite - # Prettiness - "typer>=0.20.0", - "termcolor==2.3.0", - "pytablewriter", - "rich", - "colorlog", - # Extension of metrics - "aenum==3.1.15", - # Base metrics - "nltk==3.9.1", - "scikit-learn", - "sacrebleu", - "rouge_score==0.1.2", - "sentencepiece>=0.1.99", - "protobuf", - "pycountry", - "fsspec>=2023.12.2", - "httpx>=0.27.2", - "latex2sympy2_extended==1.0.6", - "langcodes", -] - -[project.optional-dependencies] -litellm = ["litellm[caching]>=1.66.0", "diskcache"] -tgi = ["text-generation>=0.7.0"] -optimum = ["optimum==1.12.0"] -quantization = ["bitsandbytes>=0.41.0", "auto-gptq>=0.4.2"] -adapters = ["peft==0.3.0"] -nanotron = [ - "nanotron", - "tensorboardX" -] -tensorboardX = ["tensorboardX"] -vllm = ["vllm>=0.10.0,<0.10.2", "ray", "more_itertools"] -sglang = ["sglang"] -quality = ["ruff>=v0.11.0","pre-commit"] -tests = ["pytest>=7.4.0","deepdiff","pip>=25.2"] -dev = ["lighteval[accelerate,quality,tests,multilingual,math,extended_tasks,vllm]"] -docs = ["hf-doc-builder", "watchdog"] -extended_tasks = [ - "langdetect", # ifeval - "openai>1.87", # llm as a judge using openai models - "tiktoken", - "emoji", "spacy", "syllapy", # ifbench - "evaluate", # slr_bench -] -s3 = ["s3fs"] -multilingual = [ - "stanza", - "spacy[ja,ko,th]>=3.8.0", - "jieba", # for chinese tokenizer - "pyvi", # for vietnamese tokenizer -] -math = ["latex2sympy2_extended==1.0.6"] -wandb = ["wandb"] -trackio = ["trackio"] - -[project.urls] -Homepage = "https://github.com/huggingface/lighteval" -Issues = "https://github.com/huggingface/lighteval/issues" -# Documentation = "" -# Changelog = "https://github.com/huggingface/lighteval/blob/master/CHANGELOG.md" - -[project.scripts] -lighteval = "lighteval.__main__:app" diff --git a/setup.py b/setup.py deleted file mode 100644 index 00b0d04cb..000000000 --- a/setup.py +++ /dev/null @@ -1,26 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from setuptools import setup - - -setup() diff --git a/src/lighteval/__init__.py b/src/lighteval/__init__.py deleted file mode 100644 index bb2455610..000000000 --- a/src/lighteval/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import importlib.metadata - - -__version__ = importlib.metadata.version(__package__ or __name__) diff --git a/src/lighteval/__main__.py b/src/lighteval/__main__.py deleted file mode 100644 index 694a76fe5..000000000 --- a/src/lighteval/__main__.py +++ /dev/null @@ -1,90 +0,0 @@ -# MIT License - -# Copyright (c) 2024 Taratra D. RAHARISON and The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -import logging -import logging.config - -import colorlog -import typer - -import lighteval.main_accelerate -import lighteval.main_baseline -import lighteval.main_custom -import lighteval.main_endpoint -import lighteval.main_inspect -import lighteval.main_nanotron -import lighteval.main_sglang -import lighteval.main_tasks -import lighteval.main_vllm - - -app = typer.Typer() -app = typer.Typer(pretty_exceptions_show_locals=False) - -logging_config = dict( # noqa C408 - version=1, - formatters={ - "c": { - "()": colorlog.ColoredFormatter, - "format": "[%(asctime)s] [%(log_color)s%(levelname)8s%(reset)s]: %(message)s (%(filename)s:%(lineno)s)", - "log_colors": { - "DEBUG": "cyan", - "INFO": "green", - "WARNING": "yellow", - "ERROR": "red", - "CRITICAL": "red,bg_white", - }, - }, - }, - handlers={"h": {"class": "logging.StreamHandler", "formatter": "c", "level": logging.INFO}}, - root={ - "handlers": ["h"], - "level": logging.INFO, - }, -) - -logging.config.dictConfig(logging_config) -logging.captureWarnings(capture=True) - -app.command(rich_help_panel="Evaluation Backends")(lighteval.main_accelerate.accelerate) -app.command(rich_help_panel="Evaluation Utils")(lighteval.main_baseline.baseline) -app.command(rich_help_panel="Evaluation Backends")(lighteval.main_nanotron.nanotron) -app.command(rich_help_panel="Evaluation Backends")(lighteval.main_vllm.vllm) -app.command(rich_help_panel="Evaluation Backends")(lighteval.main_custom.custom) -app.command(rich_help_panel="Evaluation Backends")(lighteval.main_sglang.sglang) -app.command(rich_help_panel="Evaluation Backends")(lighteval.main_inspect.eval) -app.command(rich_help_panel="EvaluationUtils")(lighteval.main_inspect.bundle) -app.add_typer( - lighteval.main_endpoint.app, - name="endpoint", - rich_help_panel="Evaluation Backends", - help="Evaluate models using some endpoint (tgi, inference endpoint, openai) as backend.", -) -app.add_typer( - lighteval.main_tasks.app, - name="tasks", - rich_help_panel="Utils", - help="List or inspect tasks.", -) - - -if __name__ == "__main__": - app() diff --git a/src/lighteval/cli_args.py b/src/lighteval/cli_args.py deleted file mode 100644 index 2bd52af31..000000000 --- a/src/lighteval/cli_args.py +++ /dev/null @@ -1,260 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -"""Common CLI argument types for LightEval main files. -This module exports pre-defined argument types to reduce redundancy across main_*.py files. -""" - -from dataclasses import dataclass -from typing import Any, Optional - -from typer import Argument, Option -from typing_extensions import Annotated - - -# Help panel names for consistent organization -HELP_PANEL_NAME_1 = "Common Parameters" -HELP_PANEL_NAME_2 = "Logging Parameters" -HELP_PANEL_NAME_3 = "Debug Parameters" -HELP_PANEL_NAME_4 = "Modeling Parameters" - - -@dataclass -class Arg: - """Base class for CLI arguments with type and default value.""" - - type: Annotated - default: Any - - -# Common Parameters (HELP_PANEL_NAME_1) -dataset_loading_processes = Arg( - type=Annotated[ - int, - Option( - help="Number of parallel processes to use for loading datasets. Higher values can speed up dataset loading but use more memory.", - rich_help_panel=HELP_PANEL_NAME_1, - ), - ], - default=1, -) - -custom_tasks = Arg( - type=Annotated[ - Optional[str], - Option( - help="Path to a Python file containing custom task definitions. The file should define a TASKS_TABLE with LightevalTaskConfig objects.", - rich_help_panel=HELP_PANEL_NAME_1, - ), - ], - default=None, -) - -num_fewshot_seeds = Arg( - type=Annotated[ - int, - Option( - help="Number of different random seeds to use for few-shot evaluation. Each seed will generate different few-shot examples, providing more robust evaluation.", - rich_help_panel=HELP_PANEL_NAME_1, - ), - ], - default=1, -) - -load_responses_from_details_date_id = Arg( - type=Annotated[ - Optional[str], - Option( - help="Load previously generated model responses from a specific evaluation run instead of running the model. Use the timestamp/date_id from a previous run's details directory.", - rich_help_panel=HELP_PANEL_NAME_1, - ), - ], - default=None, -) - -remove_reasoning_tags = Arg( - type=Annotated[ - bool, - Option( - help="Whether to remove reasoning tags from model responses before computing metrics.", - rich_help_panel=HELP_PANEL_NAME_1, - ), - ], - default=True, -) - -reasoning_tags = Arg( - type=Annotated[ - str, - Option( - help="List of reasoning tag pairs to remove from responses, formatted as a Python list of tuples.", - rich_help_panel=HELP_PANEL_NAME_1, - ), - ], - default="[('', '')]", -) - -load_tasks_multilingual = Arg( - type=Annotated[ - bool, - Option( - help="Whether to load multilingual tasks.", - rich_help_panel=HELP_PANEL_NAME_1, - ), - ], - default=False, -) - -# Logging Parameters (HELP_PANEL_NAME_2) -output_dir = Arg( - type=Annotated[ - str, - Option( - help="Directory where evaluation results and details will be saved. Supports fsspec-compliant paths (local, s3, hf hub, etc.).", - rich_help_panel=HELP_PANEL_NAME_2, - ), - ], - default="results", -) - -results_path_template = Arg( - type=Annotated[ - str | None, - Option( - help="Custom template for results file path. Available variables: {output_dir}, {org}, {model}. Example: '{output_dir}/experiments/{org}_{model}' creates results in a subdirectory.", - rich_help_panel=HELP_PANEL_NAME_2, - ), - ], - default=None, -) - -push_to_hub = Arg( - type=Annotated[ - bool, - Option( - help="Whether to push evaluation results and details to the Hugging Face Hub. Requires --results-org to be set.", - rich_help_panel=HELP_PANEL_NAME_2, - ), - ], - default=False, -) - -push_to_tensorboard = Arg( - type=Annotated[ - bool, - Option( - help="Whether to create and push TensorBoard logs to the Hugging Face Hub. Requires --results-org to be set.", - rich_help_panel=HELP_PANEL_NAME_2, - ), - ], - default=False, -) - -public_run = Arg( - type=Annotated[ - bool, - Option( - help="Whether to make the uploaded results and details public on the Hugging Face Hub. If False, datasets will be private.", - rich_help_panel=HELP_PANEL_NAME_2, - ), - ], - default=False, -) - -results_org = Arg( - type=Annotated[ - Optional[str], - Option( - help="Hugging Face organization where results will be pushed. Required when using --push-to-hub or --push-to-tensorboard.", - rich_help_panel=HELP_PANEL_NAME_2, - ), - ], - default=None, -) - -save_details = Arg( - type=Annotated[ - bool, - Option( - help="Whether to save detailed per-sample results including model inputs, outputs, and metrics. Useful for analysis and debugging.", - rich_help_panel=HELP_PANEL_NAME_2, - ), - ], - default=False, -) - -wandb = Arg( - type=Annotated[ - bool, - Option( - help="Whether to log results to Weights & Biases (wandb) or Trackio. Configure with environment variables: WANDB_PROJECT, WANDB_SPACE_ID, etc. See wandb docs for full configuration options.", - rich_help_panel=HELP_PANEL_NAME_2, - ), - ], - default=False, -) - - -# Debug Parameters (HELP_PANEL_NAME_3) -max_samples = Arg( - type=Annotated[ - Optional[int], - Option( - help="Maximum number of samples to evaluate per task. Useful for quick testing or debugging. If None, evaluates on all available samples.", - rich_help_panel=HELP_PANEL_NAME_3, - ), - ], - default=None, -) - -job_id = Arg( - type=Annotated[ - int, - Option( - help="Optional job identifier for tracking and organizing multiple evaluation runs. Useful in cluster environments.", - rich_help_panel=HELP_PANEL_NAME_3, - ), - ], - default=0, -) - - -# Common argument patterns -tasks = Arg( - type=Annotated[ - str, - Argument( - help="Comma-separated list of tasks to evaluate. Format: 'task1,task2' or 'task{|fewshot}'. Use 'lighteval tasks list' to see available tasks." - ), - ], - default=None, # Required argument, no default -) - -model_args = Arg( - type=Annotated[ - str, - Argument( - help="Model configuration in key=value format (e.g., 'pretrained=model_name,device=cuda') or path to YAML config file. See examples/model_configs/ for template files." - ), - ], - default=None, # Required argument, no default -) diff --git a/src/lighteval/data.py b/src/lighteval/data.py deleted file mode 100644 index 50e841c6c..000000000 --- a/src/lighteval/data.py +++ /dev/null @@ -1,299 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import logging -import math -from typing import Iterator - -import torch -from packaging import version -from torch.utils.data import Dataset, Subset - -from lighteval.tasks.requests import Doc - - -if version.parse(torch.__version__) >= version.parse("2.5.0"): - from torch.utils.data.distributed import DistributedSampler, _T_co -else: - from torch.utils.data.distributed import DistributedSampler - from torch.utils.data.distributed import T_co as _T_co - - -logger = logging.getLogger(__name__) - - -class DynamicBatchDataset(Dataset): - def __init__( - self, - requests: list[Doc], - num_dataset_splits: int, - ): - """This dataset class uses dynamic batching to speed up the generation. - Each request is sorted by the length of the prompt + the length of the - continuation. Then, the dataset is split into num_dataset_splits splits. - The first split will contain the longest requests, the second split will - contain the second longest requests, etc. This allows us to use dynamic - batching by starting with a small batch size and doubling it for each - split. This is much faster than using a fixed batch size for the whole - dataset. - - Args: - requests (List): A list of requests. - num_dataset_splits (int): The number of dataset splits. - """ - # sort the requests using the collate function and save the original order - enumerated_requests = list(enumerate(requests)) - sorted_enumerated_requests = sorted(enumerated_requests, key=lambda x: self._sorting_criteria(x[1])) - - self.sorted_data = [x[1] for x in sorted_enumerated_requests] - self.original_order = [x[0] for x in sorted_enumerated_requests] - - self.total_size = len(self.sorted_data) - - self.num_dataset_splits, self.splits = self.init_split_limits(num_dataset_splits) - - def init_split_limits(self, num_dataset_splits): - if num_dataset_splits >= self.total_size: - logger.warning( - f"num_dataset_splits ({num_dataset_splits}) >= total_size ({self.total_size}), setting num_dataset_splits to 1" - ) - num_dataset_splits = 1 - - split_size = math.ceil(self.total_size / num_dataset_splits) - splits_indices = [ - (ix * split_size, min((ix + 1) * split_size, self.total_size)) for ix in range(num_dataset_splits) - ] - - return num_dataset_splits, splits_indices - - def get_original_order(self, new_arr: list) -> list: - """Get the original order of the data. - - Args: - new_arr (list): Array containing any kind of data that needs to be - reset in the original order. - - Returns: - list: new_arr in the original order. - """ - original_order = [None] * self.total_size - - for original_index, v in zip(self.original_order, new_arr): - original_order[original_index] = v - - if None in original_order: - raise RuntimeError( - f"Some elements of the original order are None, meaning that len(new_arr) ({len(new_arr)}) != len(original_array) ({self.total_size})" - ) - - return original_order - - def splits_iterator(self) -> Iterator[Subset]: - """Iterator that yields the dataset splits based on the split limits. - - Yields: - Subset: A subset of the dataset. - """ - split_range = self.num_dataset_splits - if self.total_size == 0: - split_range = 0 - for i in range(split_range): - split_start, split_end = self.splits[i] - yield Subset(self, range(split_start, split_end)) - - def __getitem__(self, index) -> Doc: - """Get an item from the dataset. - - Args: - index (int): The index of the item. - - Returns: - Any: The item at the specified index. - """ - return self.sorted_data[index] - - def __len__(self) -> int: - """Get the length of current split the dataset. - All splits have the same length, except the last one which might be - shorter. - - Returns: - int: The length of the dataset. - """ - return len(self.sorted_data) - - def __iter__(self) -> Iterator[Doc]: - """Iterator that yields the items of the dataset depending on the split we - are currently in. For instance, if we are in split 0, we will get the - items from index 0 to self.split_size, if we are in split 1, we will get - the items from index self.split_size to 2 * self.split_size, etc. Used - for dynamic batching. - - Yields: - Any: The items of the dataset. - """ - for i in range(len(self)): - yield self.sorted_data[i] - - def _sorting_criteria(self, doc: Doc): - raise NotImplementedError() - - -class LoglikelihoodDataset(DynamicBatchDataset): - def _sorting_criteria(self, doc: Doc) -> int: - """Collates the input data for batching. - - the negative sign on len(toks) sorts descending - this has a few - advantages: - - time estimates will always be over not underestimates, which is - more useful for planning - - to know the size of a batch when going through the list, you - know the first one is always the batch padded context length. this - is useful to simplify the batching logic and more importantly to make - automatic adaptive batches much much easier to implement - - any OOMs will happen right away rather than near the end - - Args: - doc (Doc): The document containing query and choices. - - Returns: - int: Negative sum of query length and max choice length for sorting. - """ - len_doc_query = len(doc.query) - max_len_choices = max(len(choice) for choice in doc.choices) if doc.choices else 0 - return -(len_doc_query + max_len_choices) - - -class GenerativeTaskDataset(DynamicBatchDataset): - def init_split_limits(self, num_dataset_splits): - """Initialises the split limits based on generation parameters. - The splits are used to estimate time remaining when evaluating, and in the case of generative evaluations, to group similar samples together. - - For generative tasks, self._sorting_criteria outputs: - - a boolean (whether the generation task uses logits) - - a list (the stop sequences) - - the item length (the actual size sorting factor). - - In the current function, we create evaluation groups by generation parameters (logits and eos), so that samples with similar properties get batched together afterwards. - The samples will then be further organised by length in each split. - - Args: - num_dataset_splits (_type_): _description_ - - Returns: - _type_: _description_ - """ - if num_dataset_splits is not None: - logger.warning( - "You cannot select the number of dataset splits for a generative evaluation at the moment. Automatically inferring." - ) - - if len(self.sorted_data) > 0: - all_sorting_criterion = [self._sorting_criteria(self.sorted_data[0])[:-1]] - splits_indices = [[0, None]] - for ix, req in enumerate(self.sorted_data): - current_sorting_criteria = self._sorting_criteria(req) - current_key = current_sorting_criteria[:-1] - if current_key not in all_sorting_criterion: - all_sorting_criterion.append(current_key) - splits_indices[-1][1] = ix - splits_indices.append([ix, None]) - - # We add the last split - splits_indices[-1][1] = self.total_size - - num_dataset_splits = len(splits_indices) - splits_indices = [tuple(e) for e in splits_indices] - return num_dataset_splits, splits_indices - - def _sorting_criteria(self, doc: Doc) -> tuple[int, bool, tuple, int, int]: - """Collate function for generating batches. - - Args: - doc (Doc): The document containing generation parameters. - - Returns: - tuple: A tuple containing (num_samples, use_logits, stop_sequences, gen_length, negative_total_length). - """ - query = doc.query - gen_length = doc.generation_size - - # The generative task has no limit except the model context - if gen_length is None: - gen_length = 0 - stop_sequences = doc.stop_sequences or [] - - return ( - doc.num_samples, - doc.use_logits, - tuple(stop_sequences), - gen_length, - -(len(query) + gen_length), - ) - - -class GenerativeTaskDatasetNanotron(GenerativeTaskDataset): - def __getitem__(self, index) -> tuple[int, Doc]: - """Get an item from the dataset depending on the split we are currently in. - For instance, if we are in split 0, we will get the item at index 0, if - we are in split 1, we will get the item at index self.split_size, etc. - Used for dynamic batching. - - Args: - index (int): The index of the item. - - Returns: - Any: The item at the specified index. - """ - return index, self.sorted_data[index + self.split_start] - - -class GenDistributedSampler(DistributedSampler): - """A distributed sampler that copy the last element only when drop_last is False so we keep a small padding in the batches - as our samples are sorted by length. - """ - - def __iter__(self) -> Iterator[_T_co]: - if self.shuffle: - # deterministically shuffle based on epoch and seed - g = torch.Generator() - g.manual_seed(self.seed + self.epoch) - indices = torch.randperm(len(self.dataset), generator=g).tolist() # type: ignore[arg-type] - else: - indices = list(range(len(self.dataset))) # type: ignore[arg-type] - - if not self.drop_last: - # add extra samples to make it evenly divisible - padding_size = self.total_size - len(indices) - indices += [ - indices[-1] - ] * padding_size # This is our only change here compared to the original DistributedSampler - else: - # remove tail of data to make it evenly divisible. - indices = indices[: self.total_size] - assert len(indices) == self.total_size - - # subsample - indices = indices[self.rank : self.total_size : self.num_replicas] - assert len(indices) == self.num_samples - - return iter(indices) diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py deleted file mode 100644 index 976b21c86..000000000 --- a/src/lighteval/logging/evaluation_tracker.py +++ /dev/null @@ -1,750 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import json -import logging -import os -import re -import time -from dataclasses import asdict, is_dataclass -from datetime import datetime -from enum import Enum -from io import BytesIO -from pathlib import Path - -import torch -from datasets import Dataset, load_dataset -from datasets.utils.metadata import MetadataConfigs -from huggingface_hub import DatasetCard, DatasetCardData, HfApi, HFSummaryWriter, hf_hub_url - -from lighteval.logging.info_loggers import ( - DetailsLogger, - GeneralConfigLogger, - MetricsLogger, - TaskConfigLogger, - VersionsLogger, -) -from lighteval.utils.imports import is_package_available, not_installed_error_message -from lighteval.utils.utils import obj_to_markdown - - -logger = logging.getLogger(__name__) - -if is_package_available("nanotron"): - from nanotron.config import GeneralArgs # type: ignore - -try: - from fsspec import url_to_fs -except ImportError: - from fsspec.core import url_to_fs - - -class EnhancedJSONEncoder(json.JSONEncoder): - """Provides a proper json encoding for the loggers and trackers json dumps. - Notably manages the json encoding of dataclasses. - """ - - def default(self, o): # noqa : C901 - if is_dataclass(o): - try: - return asdict(o) # type: ignore - except Exception: - try: - return o.__dict__ - except Exception: - return str(o) - if callable(o): - if hasattr(o, "__name__"): - return o.__name__ - # https://stackoverflow.com/questions/20594193/dynamically-created-method-and-decorator-got-error-functools-partial-object-h - # partial functions don't have __name__ so we have to unwrap the wrapped function - elif hasattr(o, "func"): - return o.func.__name__ - if isinstance(o, torch.dtype): - return str(o) - if isinstance(o, Enum): - return o.name - if hasattr(o, "__str__"): - return str(o) - try: - return super().default(o) - except TypeError: - # For classes without json serialization - return type(o).__name__ - - -class EvaluationTracker: - """Tracks and manages evaluation results, metrics, and logging for model evaluations. - - The EvaluationTracker coordinates multiple specialized loggers to track different aspects of model evaluation: - - - Details Logger (DetailsLogger): Records per-sample evaluation details and predictions - - Metrics Logger (MetricsLogger): Tracks aggregate evaluation metrics and scores - - Versions Logger (VersionsLogger): Records task and dataset versions - - General Config Logger (GeneralConfigLogger): Stores overall evaluation configuration - - Task Config Logger (TaskConfigLogger): Maintains per-task configuration details - - The tracker can save results locally and optionally push them to: - - Hugging Face Hub as datasets - - TensorBoard for visualization - - Trackio or Weights & Biases for experiment tracking - - Args: - output_dir (str): Local directory to save evaluation results and logs - results_path_template (str, optional): Template for results directory structure. - Example: "{output_dir}/results/{org}_{model}" - save_details (bool, defaults to True): Whether to save detailed evaluation records - push_to_hub (bool, defaults to False): Whether to push results to HF Hub - push_to_tensorboard (bool, defaults to False): Whether to push metrics to TensorBoard - hub_results_org (str, optional): HF Hub organization to push results to - tensorboard_metric_prefix (str, defaults to "eval"): Prefix for TensorBoard metrics - public (bool, defaults to False): Whether to make Hub datasets public - nanotron_run_info (GeneralArgs, optional): Nanotron model run information - use_wandb (bool, defaults to False): Whether to log to Weights & Biases or Trackio if available - - Example: - ```python - tracker = EvaluationTracker( - output_dir="./eval_results", - push_to_hub=True, - hub_results_org="my-org", - save_details=True - ) - - # Log evaluation results - tracker.metrics_logger.add_metric("accuracy", 0.85) - tracker.details_logger.add_detail(task_name="qa", prediction="Paris") - - # Save all results - tracker.save() - ``` - """ - - def __init__( - self, - output_dir: str, - results_path_template: str | None = None, - save_details: bool = True, - push_to_hub: bool = False, - push_to_tensorboard: bool = False, - hub_results_org: str | None = "", - tensorboard_metric_prefix: str = "eval", - public: bool = False, - nanotron_run_info: "GeneralArgs" = None, - use_wandb: bool = False, - ) -> None: - """Creates all the necessary loggers for evaluation tracking.""" - self.details_logger = DetailsLogger() - self.metrics_logger = MetricsLogger() - self.versions_logger = VersionsLogger() - self.general_config_logger = GeneralConfigLogger() - self.task_config_logger = TaskConfigLogger() - - self.api = HfApi() - self.fs, self.output_dir = url_to_fs(output_dir) - - self.hub_results_org = hub_results_org # will also contain tensorboard results - if hub_results_org in ["", None] and any([push_to_hub, push_to_tensorboard]): - raise Exception( - "You need to select which org to push to, using `--results_org`, if you want to save information to the hub." - ) - - self.should_push_to_hub = push_to_hub - self.should_save_details = save_details - self.use_wandb = use_wandb - - self.should_push_results_to_tensorboard = push_to_tensorboard - self.tensorboard_repo = f"{hub_results_org}/tensorboard_logs" - self.tensorboard_metric_prefix = tensorboard_metric_prefix - self.nanotron_run_info = nanotron_run_info - self.results_path_template = results_path_template - - self.public = public - - if use_wandb is True: - try: - import trackio as wandb - - logger.warning("Trackio was found available in your environment, using it instead of wandb") - self.wandb_project = os.environ.get("WANDB_PROJECT", None) - self.space_id = os.environ.get("WANDB_SPACE_ID", None) - - wandb_kwargs = { - "space_id": self.space_id, - } - - except ImportError: - import wandb - - self.wandb_project = os.environ.get("WANDB_PROJECT", None) - wandb.login() - wandb_kwargs = {} - - if self.wandb_project is None: - raise ValueError("You need to specify the project name using the WANDB_PROJECT environment variable") - - self.wandb_run = wandb.init( - project=self.wandb_project, - resume="allow", - **wandb_kwargs, - ) - - @property - def results(self): - config_general = asdict(self.general_config_logger) - config_general["model_config"] = config_general["model_config"].model_dump() - results = { - "config_general": config_general, - "results": self.metrics_logger.metric_aggregated, - "versions": self.versions_logger.versions, - "config_tasks": self.task_config_logger.tasks_configs, - "summary_tasks": self.details_logger.compiled_details, - "summary_general": asdict(self.details_logger.compiled_details_over_all_tasks), - } - return results - - @property - def details(self): - return { - task_name: [asdict(detail) for detail in task_details] - for task_name, task_details in self.details_logger.details.items() - } - - def preview_outputs(self) -> None: - logger.info("Previewing outputs for your eval run, one per task") - from pprint import pprint - - for task_name, task_details in self.details_logger.details.items(): - logger.info(f"Task: {task_name}") - detail = task_details[0] - # We convert the detail to a markdown string - model_response = detail.model_response - metrics = detail.metric - - pprint(model_response.text) - pprint(model_response.input) - pprint(metrics) - - def save(self) -> None: - """Saves the experiment information and results to files, and to the hub if requested.""" - logger.info("Saving experiment tracker") - date_id = datetime.now().isoformat().replace(":", "-") - - results_dict = self.results - - # Create the details datasets for later upload - details_datasets: dict[str, Dataset] = {} - for task_name, task_details in self.details_logger.details.items(): - # Create a dataset from the dictionary - we force cast to str to avoid formatting problems for nested objects - dataset = Dataset.from_list([asdict(detail) for detail in task_details]) - - # We don't keep 'id' around if it's there - column_names = dataset.column_names - if "id" in dataset.column_names: - column_names = [t for t in dataset.column_names if t != "id"] - - # Sort column names to make it easier later - dataset = dataset.select_columns(sorted(column_names)) - details_datasets[task_name] = dataset - - # We save results at every case - self.save_results(date_id, results_dict) - - if self.should_save_details: - self.save_details(date_id, details_datasets) - - if self.should_push_to_hub: - self.push_to_hub( - date_id=date_id, - details=details_datasets, - results_dict=results_dict, - ) - - if self.use_wandb is True: - self.push_to_wandb( - results_dict=results_dict, - details_datasets=details_datasets, - ) - - if self.should_push_results_to_tensorboard: - self.push_to_tensorboard( - results=self.metrics_logger.metric_aggregated, details=self.details_logger.compiled_details - ) - - def push_to_wandb(self, results_dict: dict, details_datasets: dict) -> None: - # reformat the results key to replace ':' with '/' - results_dict = {k.replace(":", "/"): v for k, v in results_dict["results"].items()} - - self.wandb_run.log( - {**results_dict}, - ) - self.wandb_run.finish() - - def save_results(self, date_id: str, results_dict: dict): - if self.results_path_template is not None: - org_model_parts = self.general_config_logger.model_name.split("/") - org = org_model_parts[0] if len(org_model_parts) >= 2 else "" - model = org_model_parts[1] if len(org_model_parts) >= 2 else org_model_parts[0] - output_dir = self.output_dir - output_dir_results = Path(self.results_path_template.format(output_dir=output_dir, org=org, model=model)) - else: - output_dir_results = Path(self.output_dir) / "results" / self.general_config_logger.model_name.strip("/") - self.fs.mkdirs(output_dir_results, exist_ok=True) - output_results_file = output_dir_results / f"results_{date_id}.json" - logger.info(f"Saving results to {output_results_file}") - with self.fs.open(output_results_file, "w") as f: - f.write(json.dumps(results_dict, cls=EnhancedJSONEncoder, indent=2, ensure_ascii=False)) - - def _get_details_sub_folder(self, date_id: str): - output_dir_details = Path(self.output_dir) / "details" / self.general_config_logger.model_name.strip("/") - if date_id in ["first", "last"]: - # Get all folders in output_dir_details - if not self.fs.exists(output_dir_details): - raise FileNotFoundError(f"Details directory {output_dir_details} does not exist") - - # List all folders and filter out files - folders = [f["name"] for f in self.fs.listdir(output_dir_details) if f["type"] == "directory"] - - if not folders: - raise FileNotFoundError(f"No timestamp folders found in {output_dir_details}") - - # Parse timestamps and get first or last - date_id = max(folders) if date_id == "last" else min(folders) - return output_dir_details / date_id - - def load_details_datasets(self, date_id: str, task_names: list[str]) -> dict[str, Dataset]: - output_dir_details_sub_folder = self._get_details_sub_folder(date_id) - logger.info(f"Loading details from {output_dir_details_sub_folder}") - date_id = output_dir_details_sub_folder.name # Overwrite date_id in case of latest - details_datasets = {} - for file in self.fs.glob(str(output_dir_details_sub_folder / f"details_*_{date_id}.parquet")): - task_name = Path(file).stem.replace("details_", "").replace(f"_{date_id}", "") - if "|".join(task_name.split("|")[:-1]) not in task_names: - logger.info(f"Skipping {task_name} because it is not in the task_names list") - continue - dataset = load_dataset("parquet", data_files=file, split="train") - details_datasets[task_name] = dataset - - for task_name in task_names: - if not any(task_name.startswith(task_name) for task_name in details_datasets.keys()): - raise ValueError( - f"Task {task_name} not found in details datasets. Check the tasks to be evaluated or the date_id used to load the details ({date_id})." - ) - return details_datasets - - def save_details(self, date_id: str, details_datasets: dict[str, Dataset]): - output_dir_details_sub_folder = self._get_details_sub_folder(date_id) - self.fs.mkdirs(output_dir_details_sub_folder, exist_ok=True) - logger.info(f"Saving details to {output_dir_details_sub_folder}") - for task_name, dataset in details_datasets.items(): - output_file_details = output_dir_details_sub_folder / f"details_{task_name}_{date_id}.parquet" - with self.fs.open(str(output_file_details), "wb") as f: - dataset.to_parquet(f) - - def generate_final_dict(self) -> dict: - """Aggregates and returns all the logger's experiment information in a dictionary. - - This function should be used to gather and display said information at the end of an evaluation run. - - Returns: - dict: Dictionary containing all experiment information including config, results, versions, and summaries - """ - to_dump = { - "config_general": asdict(self.general_config_logger), - "results": self.metrics_logger.metric_aggregated, - "versions": self.versions_logger.versions, - "config_tasks": self.task_config_logger.tasks_configs, - "summary_tasks": self.details_logger.compiled_details, - "summary_general": asdict(self.details_logger.compiled_details_over_all_tasks), - } - - final_dict = { - k: {eval_name.replace("|", ":"): eval_score for eval_name, eval_score in v.items()} - for k, v in to_dump.items() - } - - return final_dict - - def push_to_hub( - self, - date_id: str, - details: dict[str, Dataset], - results_dict: dict, - ) -> None: - """Pushes the experiment details (all the model predictions for every step) to the hub.""" - sanitized_model_name = self.general_config_logger.model_name.replace("/", "__") - - # "Default" detail names are the public detail names (same as results vs private-results) - repo_id = f"{self.hub_results_org}/details_{sanitized_model_name}" - if not self.public: # if not public, we add `_private` - repo_id = f"{repo_id}_private" - - fsspec_repo_uri = f"hf://datasets/{repo_id}" - - if not self.api.repo_exists(repo_id): - self.api.create_repo(repo_id, private=not (self.public), repo_type="dataset", exist_ok=True) - logger.info(f"Repository {repo_id} not found, creating it.") - - # We upload it both as a json and a parquet file - result_file_base_name = f"results_{date_id}" - results_json = json.dumps(results_dict, cls=EnhancedJSONEncoder, indent=2, ensure_ascii=False) - url = self.api.upload_file( - repo_id=repo_id, - path_or_fileobj=BytesIO(results_json.encode("utf-8")), - path_in_repo=f"{result_file_base_name}.json", - repo_type="dataset", - ) - logger.info(f"Uploaded evaluation details to {url}") - - results_dataset = Dataset.from_dict( - {key: [json.dumps(v, cls=EnhancedJSONEncoder, indent=2)] for key, v in results_dict.items()} - ) - results_dataset.to_parquet(f"{fsspec_repo_uri}/{result_file_base_name}.parquet") - - for task_name, dataset in details.items(): - output_file_details = Path(date_id) / f"details_{task_name}_{date_id}.parquet" - dataset.to_parquet(f"{fsspec_repo_uri}/{output_file_details}") - - self.recreate_metadata_card(repo_id) - - def push_results_to_hub(self, repo_id: str, path_in_repo: str, private: bool | None = None): - repo_id = repo_id if "/" in repo_id else f"{self.hub_results_org}/{repo_id}" - private = private if private is not None else not self.public - self.api.create_repo(repo_id, private=private, repo_type="dataset", exist_ok=True) - results_json = json.dumps(self.results, cls=EnhancedJSONEncoder, indent=2, ensure_ascii=False) - self.api.upload_file( - repo_id=repo_id, - path_or_fileobj=results_json.encode(), - path_in_repo=path_in_repo, - repo_type="dataset", - ) - - def push_details_to_hub(self, repo_id: str, path_in_repo: str, private: bool | None = None): - repo_id = repo_id if "/" in repo_id else f"{self.hub_results_org}/{repo_id}" - private = private if private is not None else not self.public - self.api.create_repo(repo_id, private=private, repo_type="dataset", exist_ok=True) - for task_name, details in self.details: - details_json = "\n".join([json.dumps(detail) for detail in details]) - self.api.upload_file( - repo_id=repo_id, - path_or_fileobj=details_json.encode(), - path_in_repo=path_in_repo.format(task_name=task_name), - repo_type="dataset", - ) - - def recreate_metadata_card(self, repo_id: str) -> None: # noqa: C901 - """Fully updates the details repository metadata card for the currently evaluated model - - Args: - repo_id (str): Details dataset repository path on the hub (`org/dataset`) - """ - # Add a nice dataset card and the configuration YAML - files_in_repo = self.api.list_repo_files(repo_id=repo_id, repo_type="dataset") - results_files = [f for f in files_in_repo if ".json" in f] - parquet_files = [f for f in files_in_repo if ".parquet" in f] - - details_file_regex = re.compile(r"details_(?P.*?)_(?P\d+-\d+-\d+T.*)\.parquet$") - multiple_results = len(results_files) > 1 - - # Get last eval results date for each task (evals might be non overlapping) - last_eval_date_results = {} - for sub_file in parquet_files: - # We focus on details only - if "results_" in sub_file: - continue - - # subfile have this general format: - # `2023-09-03T10-57-04.203304/details_harness|hendrycksTest-us_foreign_policy|5_2023-09-03T10-57-04.203304.parquet` - # in the iso date, the `:` are replaced by `-` because windows does not allow `:` in their filenames - task_name = ( - details_file_regex.match(os.path.basename(sub_file)).group("task_name") # type: ignore - ) - # task_name is then equal to `leaderboard|mmlu:us_foreign_policy|5` - - # to be able to parse the filename as iso dates, we need to re-replace the `-` with `:` - # iso_date[13] = iso_date[16] = ':' - dir_name = os.path.dirname(sub_file) - iso_date = ":".join(dir_name.rsplit("-", 2)) - eval_date = datetime.fromisoformat(iso_date) - - last_eval_date_results[task_name] = ( - max(last_eval_date_results[task_name], eval_date) if task_name in last_eval_date_results else eval_date - ) - - max_last_eval_date_results = list(last_eval_date_results.values())[0] - # Now we convert them in iso-format - for task in last_eval_date_results: - if max_last_eval_date_results < last_eval_date_results[task]: - max_last_eval_date_results = last_eval_date_results[task] - last_eval_date_results[task] = last_eval_date_results[task].isoformat() - max_last_eval_date_results = max_last_eval_date_results.isoformat() - - # Add the YAML for the configs - card_metadata = MetadataConfigs() - - # Add the results config and add the result file as a parquet file - for sub_file in parquet_files: - if "results_" in sub_file: - eval_date = os.path.basename(sub_file).replace("results_", "").replace(".parquet", "") - sanitized_task = "results" - sanitized_last_eval_date_results = re.sub(r"[^\w\.]", "_", max_last_eval_date_results) - repo_file_name = os.path.basename(sub_file) - else: - filename = os.path.basename(sub_file) - - task_name_match = details_file_regex.match(filename) # type: ignore - if not task_name_match: - raise ValueError(f"Could not parse task name from filename: {filename}") - task_name = task_name_match.group("task_name") - eval_date = task_name_match.group("date") - - sanitized_task = re.sub(r"\W", "_", task_name) - sanitized_last_eval_date_results = re.sub(r"[^\w\.]", "_", last_eval_date_results[task_name]) - repo_file_name = os.path.join("**", os.path.basename(sub_file)) - - sanitized_eval_date = re.sub(r"[^\w\.]", "_", eval_date) - - if multiple_results: - if sanitized_task not in card_metadata: - card_metadata[sanitized_task] = { - "data_files": [{"split": sanitized_eval_date, "path": [repo_file_name]}] - } - else: - former_entry = card_metadata[sanitized_task] - card_metadata[sanitized_task] = { - "data_files": former_entry["data_files"] - + [{"split": sanitized_eval_date, "path": [repo_file_name]}] - } - else: - if sanitized_task in card_metadata: - raise ValueError( - f"Entry for {sanitized_task} already exists in {former_entry} for repo {repo_id} and file {sub_file}" - ) - card_metadata[sanitized_task] = { - "data_files": [{"split": sanitized_eval_date, "path": [repo_file_name]}] - } - - if sanitized_eval_date == sanitized_last_eval_date_results: - all_entry = card_metadata[sanitized_task]["data_files"] - card_metadata[sanitized_task] = { - "data_files": all_entry + [{"split": "latest", "path": [repo_file_name]}] - } - - if "results_" in sub_file: - continue - - # Special case for MMLU with a single split covering it all - # We add another config with all MMLU splits results together for easy inspection - SPECIAL_TASKS = [ - "lighteval|mmlu", - "original|mmlu", - ] - for special_task in SPECIAL_TASKS: - sanitized_special_task = re.sub(r"\W", "_", special_task) - if sanitized_special_task in sanitized_task: - task_info = task_name.split("|") - # We have few-shot infos, let's keep them in our special task name - if len(task_info) == 3: - sanitized_special_task += f"_{task_info[-1]}" - elif len(task_info) == 4: - sanitized_special_task += f"_{task_info[-2]}_{task_info[-1]}" - if sanitized_special_task not in card_metadata: - card_metadata[sanitized_special_task] = { - "data_files": [{"split": sanitized_eval_date, "path": [repo_file_name]}] - } - else: - former_entry = card_metadata[sanitized_special_task]["data_files"] - # Any entry for this split already? - try: - split_index = next( - index - for index, dictionary in enumerate(former_entry) - if dictionary.get("split", None) == sanitized_eval_date - ) - except StopIteration: - split_index = None - if split_index is None: - card_metadata[sanitized_special_task] = { - "data_files": former_entry + [{"split": sanitized_eval_date, "path": [repo_file_name]}] - } - else: - former_entry[split_index]["path"] += [repo_file_name] - card_metadata[sanitized_special_task] = {"data_files": former_entry} - - if sanitized_eval_date == sanitized_last_eval_date_results: - former_entry = card_metadata[sanitized_special_task]["data_files"] - try: - split_index = next( - index - for index, dictionary in enumerate(former_entry) - if dictionary.get("split", None) == "latest" - ) - except StopIteration: - split_index = None - if split_index is None: - card_metadata[sanitized_special_task] = { - "data_files": former_entry + [{"split": "latest", "path": [repo_file_name]}] - } - else: - former_entry[split_index]["path"] += [repo_file_name] - card_metadata[sanitized_special_task] = {"data_files": former_entry} - - # Cleanup a little the dataset card - # Get the top results - last_results_file = [f for f in results_files if max_last_eval_date_results.replace(":", "-") in f][0] - last_results_file_path = hf_hub_url(repo_id=repo_id, filename=last_results_file, repo_type="dataset") - f: Dataset = load_dataset("json", data_files=last_results_file_path, split="train") # type: ignore - results_dict = f["results"][0] - new_dictionary = {"all": results_dict} - new_dictionary.update(results_dict) - results_string = json.dumps(new_dictionary, indent=4) - - # If we are pushing to the Open LLM Leaderboard, we'll store specific data in the model card. - is_open_llm_leaderboard = repo_id.split("/")[0] == "open-llm-leaderboard" - if is_open_llm_leaderboard: - org_string = ( - "on the [Open LLM Leaderboard](https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard)." - ) - leaderboard_url = "https://huggingface.co/spaces/HuggingFaceH4/open_llm_leaderboard" - point_of_contact = "clementine@hf.co" - else: - org_string = "" - leaderboard_url = None - point_of_contact = None - - card_data = DatasetCardData( - dataset_summary=f"Dataset automatically created during the evaluation run of model " - f"[{self.general_config_logger.model_name}](https://huggingface.co/{self.general_config_logger.model_name})" - f"{org_string}.\n\n" - f"The dataset is composed of {len(card_metadata) - 1} configuration, each one corresponding to one of the evaluated task.\n\n" - f"The dataset has been created from {len(results_files)} run(s). Each run can be found as a specific split in each " - f'configuration, the split being named using the timestamp of the run.The "train" split is always pointing to the latest results.\n\n' - f'An additional configuration "results" store all the aggregated results of the run.\n\n' - f"To load the details from a run, you can for instance do the following:\n" - f'```python\nfrom datasets import load_dataset\ndata = load_dataset("{repo_id}",\n\t"{sanitized_task}",\n\tsplit="train")\n```\n\n' - f"## Latest results\n\n" - f"These are the [latest results from run {max_last_eval_date_results}]({last_results_file_path.replace('/resolve/', '/blob/')})" - f"(note that their might be results for other tasks in the repos if successive evals didn't cover the same tasks. " - f'You find each in the results and the "latest" split for each eval):\n\n' - f"```python\n{results_string}\n```", - repo_url=f"https://huggingface.co/{self.general_config_logger.model_name}", - pretty_name=f"Evaluation run of {self.general_config_logger.model_name}", - leaderboard_url=leaderboard_url, - point_of_contact=point_of_contact, - ) - - card_metadata.to_dataset_card_data(card_data) - card = DatasetCard.from_template( - card_data, - pretty_name=card_data.pretty_name, - ) - card.push_to_hub(repo_id, repo_type="dataset") - - def push_to_tensorboard( # noqa: C901 - self, results: dict[str, dict[str, float]], details: dict[str, DetailsLogger.CompiledDetail] - ): - if not is_package_available("tensorboardX"): - logger.warning(not_installed_error_message("tensorboardX")) - return - - if not is_package_available("nanotron"): - logger.warning("You cannot push results to tensorboard without having nanotron installed. Skipping") - return - - prefix = self.tensorboard_metric_prefix - - if self.nanotron_run_info is not None: - global_step = self.nanotron_run_info.step - run = f"{self.nanotron_run_info.run}_{prefix}" - else: - global_step = 0 - run = prefix - - output_dir_tb = Path(self.output_dir) / "tb" / run.strip("/") - output_dir_tb.mkdir(parents=True, exist_ok=True) - - tb_context = HFSummaryWriter( - logdir=str(output_dir_tb), - repo_id=self.tensorboard_repo, - repo_private=True, - path_in_repo="tb", - commit_every=6000, # Very long time so that we can change our files names and trigger push ourselves (see below) - ) - bench_averages = {} - for name, values in results.items(): - splited_name = name.split("|") - if len(splited_name) == 3: - _, task_name, _ = splited_name - else: - task_name = name - bench_suite = None - if ":" in task_name: - bench_suite = task_name.split(":")[0] # e.g. MMLU - logger.info(f"bench_suite {bench_suite} in {task_name}") - for metric, value in values.items(): - if "stderr" in metric: - continue - if bench_suite not in bench_averages: - bench_averages[bench_suite] = {} - bench_averages[bench_suite][metric] = bench_averages[bench_suite].get(metric, []) + [float(value)] - logger.info(f"Pushing {task_name} {values} to tensorboard") - for metric, value in values.items(): - if "stderr" in metric: - tb_context.add_scalar(f"stderr_{prefix}/{task_name}/{metric}", value, global_step=global_step) - elif bench_suite is not None: - tb_context.add_scalar( - f"{prefix}_{bench_suite}/{task_name}/{metric}", value, global_step=global_step - ) - else: - tb_context.add_scalar(f"{prefix}/{task_name}/{metric}", value, global_step=global_step) - # Tasks with subtasks - for name, values in bench_averages.items(): - for metric, values in values.items(): - logger.info(f"Pushing average {name} {metric} {sum(values) / len(values)} to tensorboard") - tb_context.add_scalar(f"{prefix}/{name}/{metric}", sum(values) / len(values), global_step=global_step) - - tb_context.add_text("eval_config", obj_to_markdown(results), global_step=global_step) - - for task_name, task_details in details.items(): - tb_context.add_text( - f"eval_details_{task_name}", - obj_to_markdown({"0": task_details}), - global_step=global_step, - ) - - # We are doing parallel evaluations of multiple checkpoints and recording the steps not in order - # This messes up with tensorboard, so the easiest is to rename files in the order of the checkpoints - # See: https://github.com/tensorflow/tensorboard/issues/5958 - # But tensorboardX don't let us control the prefix of the files (only the suffix), so we need to do it ourselves before committing the files - - # tb_context.close() # flushes the unfinished write operations - time.sleep(5) - files = os.listdir(output_dir_tb) - for file in files: - os.rename(os.path.join(output_dir_tb, file), os.path.join(output_dir_tb, f"{global_step:07d}_{file}")) - - # Now we can push to the hub - tb_context.scheduler.trigger() - logger.info( - f"Pushed to tensorboard at https://huggingface.co/{self.tensorboard_repo}/{output_dir_tb}/tensorboard" - f" at global_step {global_step}" - ) diff --git a/src/lighteval/logging/info_loggers.py b/src/lighteval/logging/info_loggers.py deleted file mode 100644 index 35500584b..000000000 --- a/src/lighteval/logging/info_loggers.py +++ /dev/null @@ -1,440 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import collections -import logging -import os -import time -from dataclasses import asdict, dataclass, field - -import git -import xxhash - -from lighteval.metrics.utils.stderr import get_stderr_function -from lighteval.models.abstract_model import ModelConfig -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.lighteval_task import LightevalTask, LightevalTaskConfig -from lighteval.tasks.requests import Doc -from lighteval.utils.imports import is_package_available - - -logger = logging.getLogger(__name__) - - -if is_package_available("nanotron"): - pass - - -@dataclass(init=False) -class GeneralConfigLogger: - """Tracks general configuration and runtime information for model evaluations. - - This logger captures key configuration parameters, model details, and timing information - to ensure reproducibility and provide insights into the evaluation process. - - Attributes: - lighteval_sha (str): Git commit SHA of lighteval used for evaluation, enabling exact version reproducibility. - Set to "?" if not in a git repository. - - num_fewshot_seeds (int): Number of random seeds used for few-shot example sampling. - - If <= 1: Single evaluation with seed=0 - - If > 1: Multiple evaluations with different few-shot samplings (HELM-style) - - max_samples (int, optional): Maximum number of samples to evaluate per task. - Only used for debugging - truncates each task's dataset. - - job_id (int, optional): Slurm job ID if running on a cluster. - Used to cross-reference with scheduler logs. - - start_time (float): Unix timestamp when evaluation started. - Automatically set during logger initialization. - - end_time (float): Unix timestamp when evaluation completed. - Set by calling log_end_time(). - - total_evaluation_time_secondes (str): Total runtime in seconds. - Calculated as end_time - start_time. - - model_config (ModelConfig): Complete model configuration settings. - Contains model architecture, tokenizer, and generation parameters. - - model_name (str): Name identifier for the evaluated model. - Extracted from model_config. - """ - - # general - lighteval_sha: str = None - num_fewshot_seeds: int = None - max_samples: int = None - job_id: int = None - start_time: float = None - end_time: float = None - total_evaluation_time_secondes: str = None - - model_config: ModelConfig = None - model_name: str = None - - def __init__(self) -> None: - """Stores the current lighteval commit for reproducibility, and starts the evaluation timer.""" - try: - repo = git.Repo(os.path.dirname(__file__).split("src")[0]) - except git.InvalidGitRepositoryError: - repo = None - - self.lighteval_sha = repo.git.rev_parse("HEAD") if repo is not None else "?" - self.start_time = time.perf_counter() - - def log_args_info( - self, - num_fewshot_seeds: int, - max_samples: int | None, - job_id: str, - ) -> None: - """Logs the information about the arguments passed to the method. - - Args: - num_fewshot_seeds (int): number of few-shot seeds. - max_samples (int | None): maximum number of samples, if None, use all the samples available. - job_id (str): job ID, used to retrieve logs. - """ - self.num_fewshot_seeds = num_fewshot_seeds - self.max_samples = max_samples - self.job_id = job_id - - def log_model_info(self, model_config: ModelConfig) -> None: - """Logs the model information. - - Args: - model_config: the model config used to initialize the model. - """ - self.model_config = model_config - self.model_name = model_config.model_name - - def log_end_time(self) -> None: - self.end_time = time.perf_counter() - self.total_evaluation_time_secondes = str(self.end_time - self.start_time) - - -@dataclass() -class DetailsLogger: - """Logger for the experiment details. - - Stores and logs experiment information both at the task and at the sample level. - - Attributes: - hashes (dict[str, list[`Hash`]): Maps each task name to the list of all its samples' [`Hash`]. - compiled_hashes (dict[str, CompiledHash): Maps each task name to its [`CompiledHas`], an aggregation of all the individual sample hashes. - details (dict[str, list[`Detail`]]): Maps each task name to the list of its samples' details. - Example: winogrande: [sample1_details, sample2_details, ...] - compiled_details (dict[str, `CompiledDetail`]): : Maps each task name to the list of its samples' compiled details. - compiled_details_over_all_tasks (CompiledDetailOverAllTasks): Aggregated details over all the tasks. - - """ - - @dataclass() - class Detail: - """Experiment details of one single example of one task. - - Attributes: - doc (Doc): The [`Doc`] object containing the current example information. - model_response (ModelResponse): The [`ModelResponse`] object containing the model response for the current example. - metric (dict): The metric scores for the current example. - Example: {"accuracy": 0.5, "f1": 0.7, "exact_match": 0.6} - """ - - doc: Doc - model_response: ModelResponse - metric: dict - - @dataclass - class CompiledDetail: - """Experiment details compiled (for all samples) over one evaluation task. - - Attributes: - hashes (dict): A dictionary version of the [`CompiledHash`] for the current task samples. - These compiled hashes allow to quickly observe whether there is a difference between the aggregated - prompts or generations between two evaluation runs for a given task. - truncated (int): Total number of samples which needed prompt truncation to fit the model context size for the current task. - non_truncated (int): Total number of samples which did not need prompt truncation to fit the model context size for the current task. - padded (int): Total umber of samples which needed padding during the batching step for the current task. - non_padded (int): Total number of samples which did not need padding during the batching step for the current task. - """ - - hashes: dict = field(default_factory=dict) - truncated: int = 0 - non_truncated: int = 0 - padded: int = 0 - non_padded: int = 0 - - @dataclass - class CompiledDetailOverAllTasks: - """Experiment details compiled across all evaluation tasks. - - Attributes: - hashes (dict): For each key, average hash of the [`CompiledHash`] values across all tasks. - These compiled hashes allow to quickly observe whether there is a difference between the aggregated - prompts or generations between two evaluation runs across all tasks. - truncated (int): Total number of samples which needed prompt truncation to fit the model context size across all tasks. - non_truncated (int): Total number of samples which did not need prompt truncation to fit the model context size across all tasks - padded (int): Number of samples which needed padding during the batching step across all tasks. - non_padded (int): Number of samples which did not need padding during the batching step across all tasks. - """ - - hashes: dict = field(default_factory=dict) - truncated: int = 0 - non_truncated: int = 0 - padded: int = 0 - non_padded: int = 0 - - @dataclass - class Hash: - """Hashes important values for one sample ([`Doc`]) of one task ([`LightevalTask`]) - - Attributes: - example (str): Hash of the [`Doc.query`] - full_prompt (str): Hash of the [`Doc.ctx`] - input_tokens (str): Aggregated hash of all the [`Doc.input_tokens`] - cont_tokens (str): Aggregated hash of all the [`Doc.generated_tokens`] - - """ - - example: str = "" - full_prompt: str = "" - input_tokens: str = "" - cont_tokens: str = "" - - @dataclass - class CompiledHash: - """Hashes the aggregated hash values for all the sample ([`Doc`]) of one task ([`LightevalTask`]) - - Attributes: - example (str): Aggregated hash of all the [`Doc.query`] hashes for all samples of the current task. - full_prompt (str): Aggregated hash of all the [`Doc.ctx`] hashes for all samples of the current task. - input_tokens (str): Aggregated hash of the aggregated [`Doc.input_tokens`] hashes over all samples of the current task. - cont_tokens (str): Aggregated hash of the aggregated [`Doc.generated_tokens`] hashes over all samples of the current task. - """ - - hash_examples: str = "" - hash_full_prompts: str = "" - hash_input_tokens: str = "" - hash_cont_tokens: str = "" - - hashes: dict[str, list[Hash]] = field(default_factory=lambda: collections.defaultdict(list)) - compiled_hashes: dict[str, CompiledHash] = field( - default_factory=lambda: collections.defaultdict(DetailsLogger.CompiledHash) - ) - - # dict of details for each task, i.e. winogrande: [example1_details, example2_details, ...] - details: dict[str, list[Detail]] = field(default_factory=lambda: collections.defaultdict(list)) - compiled_details: dict[str, CompiledDetail] = field( - default_factory=lambda: collections.defaultdict(DetailsLogger.CompiledDetail) - ) - compiled_details_over_all_tasks: CompiledDetailOverAllTasks = field(default_factory=CompiledDetailOverAllTasks) - - def log( - self, - task_name: str, - doc: Doc, - model_response: ModelResponse, - metrics: dict, - ) -> None: - """Stores the relevant information for one sample of one task to the total list of samples stored in the DetailsLogger. - - Args: - task_name (str): Name of the current task of interest. - doc (Doc): Current sample that we want to store. - model_response (ModelResponse): Model outputs for the current sample - metrics (dict): Model scores for said sample on the current task's metrics. - """ - detail = self.Detail(doc, model_response, metrics) - self.details[task_name].append(detail) - - hash = self.Hash() - hash.example = xxhash.xxh64(doc.query).hexdigest() - hash.input_tokens = xxhash.xxh64(str(model_response.input_tokens)).hexdigest() - hash.cont_tokens = xxhash.xxh64(str(model_response.output_tokens)).hexdigest() - self.hashes[task_name].append(hash) - - def aggregate(self): - """Hashes the details for each task and then for all tasks.""" - for task_name in self.hashes: - compiled_hash = self.CompiledHash() - compiled_hash.hash_examples = xxhash.xxh64( - "".join(sorted(q.example for q in self.hashes[task_name])) - ).hexdigest() # hash of all the hash - sorted for reproducibility - compiled_hash.hash_full_prompts = xxhash.xxh64( - "".join(sorted(q.full_prompt for q in self.hashes[task_name])) - ).hexdigest() # hash of all the hash - sorted for reproducibility - compiled_hash.hash_input_tokens = xxhash.xxh64( - "".join(sorted(q.input_tokens for q in self.hashes[task_name])) - ).hexdigest() # hash of all the hash - sorted for reproducibility - compiled_hash.hash_cont_tokens = xxhash.xxh64( - "".join(sorted(q.cont_tokens for q in self.hashes[task_name])) - ).hexdigest() # hash of all the hash - sorted for reproducibility - self.compiled_hashes[task_name] = compiled_hash - - for task_name, _ in self.details.items(): - self.compiled_details[task_name].hashes = asdict(self.compiled_hashes[task_name]) - - hash_types: list[str] = list(self.compiled_details.values())[0].hashes.keys() - - for hash_type in hash_types: - self.compiled_details_over_all_tasks.hashes[hash_type] = xxhash.xxh64( - "".join( - compiled_detail.hashes[hash_type] for _, compiled_detail in sorted(self.compiled_details.items()) - ) - ).hexdigest() - - -@dataclass -class MetricsLogger: - """Logs the actual scores for each metric of each task. - - Attributes: - metrics_value (dict[str, dict[str, list[float]]]): Maps each task to its dictionary of metrics to scores for all the example of the task. - Example: {"winogrande|winogrande_xl": {"accuracy": [0.5, 0.5, 0.5, 0.5, 0.5, 0.5]}} - metric_aggregated (dict[str, dict[str, float]]): Maps each task to its dictionary of metrics to aggregated scores over all the example of the task. - Example: {"winogrande|winogrande_xl": {"accuracy": 0.5}} - """ - - metrics_values: dict[str, dict[str, list[float]]] = field( - default_factory=lambda: collections.defaultdict(lambda: collections.defaultdict(list)) - ) - metric_aggregated: dict[str, dict[str, float]] = field( - default_factory=lambda: collections.defaultdict(lambda: collections.defaultdict(float)) - ) - - def log(self, task_name: str, metrics: dict) -> None: - for metric_name, metric_value in metrics.items(): - self.metrics_values[task_name][metric_name].append(metric_value) - - def aggregate(self, task_dict: dict[str, LightevalTask], bootstrap_iters: int = 1000): # noqa: C901 - """Aggregate the metrics for each task and then for all tasks. - - Args: - task_dict (dict[str, LightevalTask]): used to determine what aggregation function to use for each metric - bootstrap_iters (int, optional): Number of runs used to run the statistical bootstrap. Defaults to 1000. - """ - for task_name, metrics in self.metrics_values.items(): - task = task_dict[task_name] - - skip_metric = [] - for metric_name, metric_values in metrics.items(): - if metric_name in skip_metric: - # The metric is in a subset which has already been computed and saved - continue - - aggregation = task.aggregation()[metric_name] - - try: - metric_result = aggregation(metric_values) - except OverflowError: - logger.warning(f"{task_name}, {metric_name} got an OVERFLOW ERROR when aggregating.") - metric_result = float("nan") - - if isinstance(metric_result, dict): # For some corpus level grouping metrics - self.metric_aggregated[task_name].update(metric_result) - skip_metric.extend(list(metric_result.keys())) # no need to recompute them later - else: - self.metric_aggregated[task_name][metric_name] = metric_result - - if isinstance(metric_result, dict) or bootstrap_iters == 0: - stderr = ( - None # We skip stderr for some corpus metrics that return dicts, or if bootstrap_iters is 0 - ) - else: - stderr = get_stderr_function(aggregation=aggregation, number_experiments=bootstrap_iters) - if stderr is not None and len(metric_values) > 1: - try: - self.metric_aggregated[task_name][f"{metric_name}_stderr"] = stderr(metric_values) - except OverflowError: - # Is this need or should we just pass? - self.metric_aggregated[task_name][f"{metric_name}_stderr"] = float("nan") - logger.warning(f"{task_name}, {metric_name} got an OVERFLOW ERROR when computing stderr.") - - # We group subtasks which belong to the same parent task, like MMLU, to compute an average on them - # and compute an average of all metrics - grouped_tasks = collections.defaultdict(list) - suite_average = {} - suite_nb = {} - - # Build aggregation - for k, metrics in self.metric_aggregated.items(): - if "|" in k: - task, fewshot = k.split("|") - grouped_tasks[f"{task.split(':')[0]}:_average|{fewshot}"].append(k) - for metric, value in metrics.items(): - suite_average[metric] = suite_average.get(metric, 0) + value - suite_nb[metric] = suite_nb.get(metric, 0) + 1 - - # Compute average for sub groups - for average_task, list_of_subtasks in grouped_tasks.items(): - if len(list_of_subtasks) > 1: - metrics = list(self.metric_aggregated[list_of_subtasks[0]].keys()) - self.metric_aggregated[average_task] = { - metric: sum(self.metric_aggregated[k][metric] for k in list_of_subtasks) / len(list_of_subtasks) - for metric in metrics - } - - # Compute average for all - for metric, value in suite_average.items(): - suite_average[metric] = value / suite_nb[metric] - - self.metric_aggregated["all"] = suite_average - - -@dataclass -class VersionsLogger: - """Logger of the tasks versions. - - Tasks can have a version number/date, which indicates what is the precise metric definition and dataset version used for an evaluation. - - Attributes: - version (dict[str, int]): Maps the task names with the task versions. - - """ - - # the versions dict will be a dict of task_name: task_version - # {"winogrande|winogrande_xl": 0} - versions: dict[str, int] = field(default_factory=dict) - - def log(self, task_name: str, task_version: int) -> None: - self.versions[task_name] = task_version - - -@dataclass -class TaskConfigLogger: - """Logs the different parameters of the current [`LightevalTask`] of interest. - - Attributes: - tasks_config (dict[str, LightevalTaskConfig]): Maps each task to its associated [`LightevalTaskConfig`] - - """ - - tasks_configs: dict[str, LightevalTaskConfig] = field(default_factory=dict) - - def log(self, task_dict: dict[str, LightevalTask]) -> None: - self.tasks_configs = {name: task.config for name, task in task_dict.items()} - - def log_num_docs(self, task_name: str, original_num_docs: int, effective_num_docs: int) -> None: - self.tasks_configs[task_name].original_num_docs = original_num_docs - self.tasks_configs[task_name].effective_num_docs = effective_num_docs diff --git a/src/lighteval/main_accelerate.py b/src/lighteval/main_accelerate.py deleted file mode 100644 index 00fe25676..000000000 --- a/src/lighteval/main_accelerate.py +++ /dev/null @@ -1,156 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import logging - -from typer import Option -from typing_extensions import Annotated - -from lighteval.cli_args import ( - HELP_PANEL_NAME_4, - custom_tasks, - dataset_loading_processes, - job_id, - load_responses_from_details_date_id, - load_tasks_multilingual, - max_samples, - model_args, - num_fewshot_seeds, - output_dir, - public_run, - push_to_hub, - push_to_tensorboard, - reasoning_tags, - remove_reasoning_tags, - results_org, - results_path_template, - save_details, - tasks, - wandb, -) - - -logger = logging.getLogger(__name__) - - -def accelerate( # noqa C901 - # === general === - model_args: model_args.type, - tasks: tasks.type, - # === Common parameters === - vision_model: Annotated[ - bool, Option(help="Use vision model for evaluation.", rich_help_panel=HELP_PANEL_NAME_4) - ] = False, - custom_tasks: custom_tasks.type = custom_tasks.default, - load_tasks_multilingual: load_tasks_multilingual.type = load_tasks_multilingual.default, - dataset_loading_processes: dataset_loading_processes.type = dataset_loading_processes.default, - num_fewshot_seeds: num_fewshot_seeds.type = num_fewshot_seeds.default, - load_responses_from_details_date_id: load_responses_from_details_date_id.type = load_responses_from_details_date_id.default, - remove_reasoning_tags: remove_reasoning_tags.type = remove_reasoning_tags.default, - reasoning_tags: reasoning_tags.type = reasoning_tags.default, - # === saving === - output_dir: output_dir.type = output_dir.default, - results_path_template: results_path_template.type = results_path_template.default, - push_to_hub: push_to_hub.type = push_to_hub.default, - push_to_tensorboard: push_to_tensorboard.type = push_to_tensorboard.default, - public_run: public_run.type = public_run.default, - results_org: results_org.type = results_org.default, - save_details: save_details.type = save_details.default, - wandb: wandb.type = wandb.default, - # === debug === - max_samples: max_samples.type = max_samples.default, - job_id: job_id.type = job_id.default, -): - """Evaluate models using accelerate and transformers as backend. - - Returns: - dict: Evaluation results containing metrics and scores for all tasks - """ - import yaml - - from lighteval.logging.evaluation_tracker import EvaluationTracker - from lighteval.models.abstract_model import ModelConfig - from lighteval.models.transformers.adapter_model import AdapterModelConfig - from lighteval.models.transformers.delta_model import DeltaModelConfig - from lighteval.models.transformers.transformers_model import TransformersModelConfig - from lighteval.models.transformers.vlm_transformers_model import VLMTransformersModelConfig - from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters - - evaluation_tracker = EvaluationTracker( - output_dir=output_dir, - results_path_template=results_path_template, - save_details=save_details, - push_to_hub=push_to_hub, - push_to_tensorboard=push_to_tensorboard, - public=public_run, - hub_results_org=results_org, - use_wandb=wandb, - ) - pipeline_params = PipelineParameters( - launcher_type=ParallelismManager.ACCELERATE, - custom_tasks_directory=custom_tasks, - load_tasks_multilingual=load_tasks_multilingual, - job_id=job_id, - dataset_loading_processes=dataset_loading_processes, - num_fewshot_seeds=num_fewshot_seeds, - max_samples=max_samples, - remove_reasoning_tags=remove_reasoning_tags, - reasoning_tags=reasoning_tags, - load_responses_from_details_date_id=load_responses_from_details_date_id, - ) - - if model_args.endswith(".yaml"): - with open(model_args, "r") as f: - config = yaml.safe_load(f)["model_parameters"] - else: - # We extract the model args - config: dict = ModelConfig._parse_args(model_args) - - if config.get("delta_weights", False): - config.pop("delta_weights") - model_config = DeltaModelConfig(**config) - elif config.get("adapter_weights", False): - config.pop("adapter_weights") - model_config = AdapterModelConfig(**config) - else: - if vision_model: - model_config = VLMTransformersModelConfig(**config) - else: - model_config = TransformersModelConfig(**config) - - pipeline = Pipeline( - tasks=tasks, - pipeline_parameters=pipeline_params, - evaluation_tracker=evaluation_tracker, - model_config=model_config, - ) - - pipeline.evaluate() - - pipeline.show_results() - - results = pipeline.get_results() - details = pipeline.get_details() - - pipeline.save_and_push_results() - - return results, details diff --git a/src/lighteval/main_baseline.py b/src/lighteval/main_baseline.py deleted file mode 100644 index 2ba82095c..000000000 --- a/src/lighteval/main_baseline.py +++ /dev/null @@ -1,94 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -from lighteval.cli_args import ( - custom_tasks, - dataset_loading_processes, - load_tasks_multilingual, - max_samples, - output_dir, - tasks, -) - - -def baseline( - tasks: tasks.type, - load_tasks_multilingual: load_tasks_multilingual.type = load_tasks_multilingual.default, - dataset_loading_processes: dataset_loading_processes.type = dataset_loading_processes.default, - custom_tasks: custom_tasks.type = custom_tasks.default, - output_dir: output_dir.type = output_dir.default, - max_samples: max_samples.type = max_samples.default, -): - """Compute baselines for given tasks. - - It has been tested with generative and accuracy tasks, but may not work correctly for other task types. - - The baseline is computed as follows: - - - For multiple-choice tasks: It assumes random guessing, so the score is n_correct/number_of_choices. - - For other metrics: It assigns a score of 0, which may not be appropriate for all task types. - - Note: - This baseline computation may not be suitable for all task types and should be used with caution. - """ - from lighteval.logging.evaluation_tracker import EvaluationTracker - from lighteval.tasks.lighteval_task import LightevalTask - from lighteval.tasks.registry import Registry - from lighteval.tasks.requests import SamplingMethod - from lighteval.utils.utils import as_list - - registry = Registry(tasks=tasks, custom_tasks=custom_tasks, load_multilingual=load_tasks_multilingual) - tasks_dict: dict[str, LightevalTask] = registry.load_tasks() - - evaluation_tracker = EvaluationTracker( - output_dir=output_dir, - save_details=False, - push_to_hub=False, - push_to_tensorboard=False, - public=False, - hub_results_org=None, - ) - evaluation_tracker.general_config_logger.log_model_info( - model_config=None, - ) - evaluation_tracker.task_config_logger.log(tasks_dict) - - LightevalTask.load_datasets(tasks_dict, dataset_loading_processes) - - for task_name, task in tasks_dict.items(): - task_docs = list(task.eval_docs()) - n_samples = min(max_samples, len(task_docs)) if max_samples else len(task_docs) - - p_correct_score = [ - len(as_list(task_doc.gold_index)) / len(task_doc.choices) for task_doc in task_docs[:n_samples] - ] - - metric_results = { - metric.metric_name: p_correct_score if metric.category in [SamplingMethod.LOGPROBS] else 0 - for metric in task.metrics - } - - evaluation_tracker.metrics_logger.log(task_name, metric_results) - - evaluation_tracker.metrics_logger.aggregate(task_dict=tasks_dict, bootstrap_iters=1000) - evaluation_tracker.save() diff --git a/src/lighteval/main_custom.py b/src/lighteval/main_custom.py deleted file mode 100644 index e6124ce62..000000000 --- a/src/lighteval/main_custom.py +++ /dev/null @@ -1,124 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -import typer -from typer import Argument -from typing_extensions import Annotated - -from lighteval.cli_args import ( - custom_tasks, - dataset_loading_processes, - job_id, - load_tasks_multilingual, - max_samples, - num_fewshot_seeds, - output_dir, - public_run, - push_to_hub, - push_to_tensorboard, - reasoning_tags, - remove_reasoning_tags, - results_org, - results_path_template, - save_details, - tasks, -) -from lighteval.models.custom.custom_model import CustomModelConfig - - -app = typer.Typer() - - -@app.command(rich_help_panel="Evaluation Backends") -def custom( - # === general === - model_name: Annotated[str, Argument(help="The model name to evaluate")], - model_definition_file_path: Annotated[str, Argument(help="The model definition file path to evaluate")], - tasks: tasks.type, - # === Common parameters === - load_tasks_multilingual: load_tasks_multilingual.type = load_tasks_multilingual.default, - dataset_loading_processes: dataset_loading_processes.type = dataset_loading_processes.default, - num_fewshot_seeds: num_fewshot_seeds.type = num_fewshot_seeds.default, - custom_tasks: custom_tasks.type = custom_tasks.default, - remove_reasoning_tags: remove_reasoning_tags.type = remove_reasoning_tags.default, - reasoning_tags: reasoning_tags.type = reasoning_tags.default, - # === saving === - output_dir: output_dir.type = output_dir.default, - results_path_template: results_path_template.type = results_path_template.default, - push_to_hub: push_to_hub.type = push_to_hub.default, - push_to_tensorboard: push_to_tensorboard.type = push_to_tensorboard.default, - public_run: public_run.type = public_run.default, - results_org: results_org.type = results_org.default, - save_details: save_details.type = save_details.default, - # === debug === - max_samples: max_samples.type = max_samples.default, - job_id: job_id.type = job_id.default, -): - """Evaluate custom models (can be anything). - - Returns: - dict: Evaluation results containing metrics and scores for all tasks - """ - from lighteval.logging.evaluation_tracker import EvaluationTracker - from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters - - evaluation_tracker = EvaluationTracker( - output_dir=output_dir, - results_path_template=results_path_template, - save_details=save_details, - push_to_hub=push_to_hub, - push_to_tensorboard=push_to_tensorboard, - public=public_run, - hub_results_org=results_org, - ) - - parallelism_manager = ParallelismManager.CUSTOM - model_config = CustomModelConfig(model_name=model_name, model_definition_file_path=model_definition_file_path) - - pipeline_params = PipelineParameters( - launcher_type=parallelism_manager, - job_id=job_id, - dataset_loading_processes=dataset_loading_processes, - custom_tasks_directory=custom_tasks, - num_fewshot_seeds=num_fewshot_seeds, - max_samples=max_samples, - remove_reasoning_tags=remove_reasoning_tags, - reasoning_tags=reasoning_tags, - load_tasks_multilingual=load_tasks_multilingual, - ) - pipeline = Pipeline( - tasks=tasks, - pipeline_parameters=pipeline_params, - evaluation_tracker=evaluation_tracker, - model_config=model_config, - ) - - pipeline.evaluate() - - pipeline.show_results() - - results = pipeline.get_results() - - pipeline.save_and_push_results() - - return results diff --git a/src/lighteval/main_endpoint.py b/src/lighteval/main_endpoint.py deleted file mode 100644 index ece2ac430..000000000 --- a/src/lighteval/main_endpoint.py +++ /dev/null @@ -1,409 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -import typer -from typer import Argument, Option -from typing_extensions import Annotated - -from lighteval.cli_args import ( - HELP_PANEL_NAME_4, - custom_tasks, - dataset_loading_processes, - job_id, - load_responses_from_details_date_id, - load_tasks_multilingual, - max_samples, - num_fewshot_seeds, - output_dir, - public_run, - push_to_hub, - push_to_tensorboard, - reasoning_tags, - remove_reasoning_tags, - results_org, - results_path_template, - save_details, - tasks, - wandb, -) - - -app = typer.Typer() - - -@app.command(rich_help_panel="Evaluation Backends") -def inference_endpoint( - # === general === - model_config_path: Annotated[ - str, Argument(help="Path to model config yaml file. (examples/model_configs/endpoint_model.yaml)") - ], - tasks: tasks.type, - free_endpoint: Annotated[ - bool, - Option( - help="Use serverless free endpoints instead of spinning up your own inference endpoint.", - rich_help_panel=HELP_PANEL_NAME_4, - ), - ] = False, - # === Common parameters === - load_tasks_multilingual: load_tasks_multilingual.type = load_tasks_multilingual.default, - dataset_loading_processes: dataset_loading_processes.type = dataset_loading_processes.default, - custom_tasks: custom_tasks.type = custom_tasks.default, - num_fewshot_seeds: num_fewshot_seeds.type = num_fewshot_seeds.default, - load_responses_from_details_date_id: load_responses_from_details_date_id.type = load_responses_from_details_date_id.default, - remove_reasoning_tags: remove_reasoning_tags.type = remove_reasoning_tags.default, - reasoning_tags: reasoning_tags.type = reasoning_tags.default, - # === saving === - output_dir: output_dir.type = output_dir.default, - results_path_template: results_path_template.type = results_path_template.default, - push_to_hub: push_to_hub.type = push_to_hub.default, - push_to_tensorboard: push_to_tensorboard.type = push_to_tensorboard.default, - public_run: public_run.type = public_run.default, - results_org: results_org.type = results_org.default, - save_details: save_details.type = save_details.default, - wandb: wandb.type = wandb.default, - # === debug === - max_samples: max_samples.type = max_samples.default, - job_id: job_id.type = job_id.default, -): - """Evaluate models using inference-endpoints as backend. - - Returns: - dict: Evaluation results containing metrics and scores for all tasks - """ - from lighteval.logging.evaluation_tracker import EvaluationTracker - from lighteval.models.endpoints.endpoint_model import InferenceEndpointModelConfig, ServerlessEndpointModelConfig - from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters - - evaluation_tracker = EvaluationTracker( - output_dir=output_dir, - results_path_template=results_path_template, - save_details=save_details, - push_to_hub=push_to_hub, - push_to_tensorboard=push_to_tensorboard, - public=public_run, - hub_results_org=results_org, - use_wandb=wandb, - ) - - parallelism_manager = ParallelismManager.NONE # since we're using inference endpoints in remote - - if free_endpoint: - model_config = ServerlessEndpointModelConfig.from_path(model_config_path) - else: - model_config = InferenceEndpointModelConfig.from_path(model_config_path) - - pipeline_params = PipelineParameters( - launcher_type=parallelism_manager, - job_id=job_id, - dataset_loading_processes=dataset_loading_processes, - custom_tasks_directory=custom_tasks, - num_fewshot_seeds=num_fewshot_seeds, - max_samples=max_samples, - load_responses_from_details_date_id=load_responses_from_details_date_id, - remove_reasoning_tags=remove_reasoning_tags, - reasoning_tags=reasoning_tags, - load_tasks_multilingual=load_tasks_multilingual, - ) - pipeline = Pipeline( - tasks=tasks, - pipeline_parameters=pipeline_params, - evaluation_tracker=evaluation_tracker, - model_config=model_config, - ) - - pipeline.evaluate() - - pipeline.show_results() - - results = pipeline.get_results() - - pipeline.save_and_push_results() - - return results - - -@app.command(rich_help_panel="Evaluation Backends") -def tgi( - # === general === - model_config_path: Annotated[ - str, Argument(help="Path to model config yaml file. (examples/model_configs/tgi_model.yaml)") - ], - tasks: tasks.type, - # === Common parameters === - load_tasks_multilingual: load_tasks_multilingual.type = load_tasks_multilingual.default, - dataset_loading_processes: dataset_loading_processes.type = dataset_loading_processes.default, - custom_tasks: custom_tasks.type = custom_tasks.default, - num_fewshot_seeds: num_fewshot_seeds.type = num_fewshot_seeds.default, - load_responses_from_details_date_id: load_responses_from_details_date_id.type = load_responses_from_details_date_id.default, - remove_reasoning_tags: remove_reasoning_tags.type = remove_reasoning_tags.default, - reasoning_tags: reasoning_tags.type = reasoning_tags.default, - # === saving === - output_dir: output_dir.type = output_dir.default, - results_path_template: results_path_template.type = results_path_template.default, - push_to_hub: push_to_hub.type = push_to_hub.default, - push_to_tensorboard: push_to_tensorboard.type = push_to_tensorboard.default, - public_run: public_run.type = public_run.default, - results_org: results_org.type = results_org.default, - save_details: save_details.type = save_details.default, - wandb: wandb.type = wandb.default, - # === debug === - max_samples: max_samples.type = max_samples.default, - job_id: job_id.type = job_id.default, -): - """Evaluate models using TGI as backend. - - Returns: - dict: Evaluation results containing metrics and scores for all tasks - """ - from lighteval.logging.evaluation_tracker import EvaluationTracker - from lighteval.models.endpoints.tgi_model import TGIModelConfig - from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters - - evaluation_tracker = EvaluationTracker( - output_dir=output_dir, - results_path_template=results_path_template, - save_details=save_details, - push_to_hub=push_to_hub, - push_to_tensorboard=push_to_tensorboard, - public=public_run, - hub_results_org=results_org, - use_wandb=wandb, - ) - - parallelism_manager = ParallelismManager.TGI - - model_config = TGIModelConfig.from_path(model_config_path) - - pipeline_params = PipelineParameters( - launcher_type=parallelism_manager, - load_tasks_multilingual=load_tasks_multilingual, - job_id=job_id, - dataset_loading_processes=dataset_loading_processes, - custom_tasks_directory=custom_tasks, - num_fewshot_seeds=num_fewshot_seeds, - max_samples=max_samples, - load_responses_from_details_date_id=load_responses_from_details_date_id, - remove_reasoning_tags=remove_reasoning_tags, - reasoning_tags=reasoning_tags, - ) - pipeline = Pipeline( - tasks=tasks, - pipeline_parameters=pipeline_params, - evaluation_tracker=evaluation_tracker, - model_config=model_config, - ) - - pipeline.evaluate() - - pipeline.show_results() - - results = pipeline.get_results() - - pipeline.save_and_push_results() - - return results - - -@app.command(rich_help_panel="Evaluation Backends") -def litellm( - # === general === - model_args: Annotated[ - str, - Argument( - help="config file path for the litellm model, or a comma separated string of model args (model_name={},base_url={},provider={})" - ), - ], - tasks: tasks.type, - # === Common parameters === - load_tasks_multilingual: load_tasks_multilingual.type = load_tasks_multilingual.default, - dataset_loading_processes: dataset_loading_processes.type = dataset_loading_processes.default, - num_fewshot_seeds: num_fewshot_seeds.type = num_fewshot_seeds.default, - custom_tasks: custom_tasks.type = custom_tasks.default, - load_responses_from_details_date_id: load_responses_from_details_date_id.type = load_responses_from_details_date_id.default, - remove_reasoning_tags: remove_reasoning_tags.type = remove_reasoning_tags.default, - reasoning_tags: reasoning_tags.type = reasoning_tags.default, - # === saving === - output_dir: output_dir.type = output_dir.default, - results_path_template: results_path_template.type = results_path_template.default, - push_to_hub: push_to_hub.type = push_to_hub.default, - push_to_tensorboard: push_to_tensorboard.type = push_to_tensorboard.default, - public_run: public_run.type = public_run.default, - results_org: results_org.type = results_org.default, - save_details: save_details.type = save_details.default, - wandb: wandb.type = wandb.default, - # === debug === - max_samples: max_samples.type = max_samples.default, - job_id: job_id.type = job_id.default, -): - """Evaluate models using LiteLLM as backend. - - Returns: - dict: Evaluation results containing metrics and scores for all tasks - """ - import yaml - - from lighteval.logging.evaluation_tracker import EvaluationTracker - from lighteval.models.endpoints.litellm_model import LiteLLMModelConfig - from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters - - evaluation_tracker = EvaluationTracker( - output_dir=output_dir, - results_path_template=results_path_template, - save_details=save_details, - push_to_hub=push_to_hub, - push_to_tensorboard=push_to_tensorboard, - public=public_run, - hub_results_org=results_org, - use_wandb=wandb, - ) - - parallelism_manager = ParallelismManager.NONE - - if model_args.endswith(".yaml"): - with open(model_args, "r") as f: - config = yaml.safe_load(f) - metric_options = config.get("metric_options", {}) - model_config = LiteLLMModelConfig.from_path(model_args) - else: - metric_options = None - model_config = LiteLLMModelConfig.from_args(model_args) - - pipeline_params = PipelineParameters( - launcher_type=parallelism_manager, - load_tasks_multilingual=load_tasks_multilingual, - job_id=job_id, - dataset_loading_processes=dataset_loading_processes, - custom_tasks_directory=custom_tasks, - num_fewshot_seeds=num_fewshot_seeds, - max_samples=max_samples, - load_responses_from_details_date_id=load_responses_from_details_date_id, - remove_reasoning_tags=remove_reasoning_tags, - reasoning_tags=reasoning_tags, - ) - pipeline = Pipeline( - tasks=tasks, - pipeline_parameters=pipeline_params, - evaluation_tracker=evaluation_tracker, - model_config=model_config, - metric_options=metric_options, - ) - - pipeline.evaluate() - - pipeline.show_results() - - results = pipeline.get_results() - - pipeline.save_and_push_results() - - return results - - -@app.command(rich_help_panel="Evaluation Backends") -def inference_providers( - # === general === - model_args: Annotated[ - str, - Argument( - help="config file path for the inference provider model, or a comma separated string of model args (model_name={},provider={},generation={temperature: 0.6})" - ), - ], - tasks: tasks.type, - # === Common parameters === - load_tasks_multilingual: load_tasks_multilingual.type = load_tasks_multilingual.default, - dataset_loading_processes: dataset_loading_processes.type = dataset_loading_processes.default, - custom_tasks: custom_tasks.type = custom_tasks.default, - num_fewshot_seeds: num_fewshot_seeds.type = num_fewshot_seeds.default, - # === saving === - output_dir: output_dir.type = output_dir.default, - results_path_template: results_path_template.type = results_path_template.default, - push_to_hub: push_to_hub.type = push_to_hub.default, - push_to_tensorboard: push_to_tensorboard.type = push_to_tensorboard.default, - public_run: public_run.type = public_run.default, - results_org: results_org.type = results_org.default, - save_details: save_details.type = save_details.default, - wandb: wandb.type = wandb.default, - remove_reasoning_tags: remove_reasoning_tags.type = remove_reasoning_tags.default, - reasoning_tags: reasoning_tags.type = reasoning_tags.default, - # === debug === - max_samples: max_samples.type = max_samples.default, - job_id: job_id.type = job_id.default, -): - """Evaluate models using HuggingFace's inference providers as backend. - - Returns: - dict: Evaluation results containing metrics and scores for all tasks - """ - from lighteval.logging.evaluation_tracker import EvaluationTracker - from lighteval.models.endpoints.inference_providers_model import ( - InferenceProvidersModelConfig, - ) - from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters - - evaluation_tracker = EvaluationTracker( - output_dir=output_dir, - results_path_template=results_path_template, - save_details=save_details, - push_to_hub=push_to_hub, - push_to_tensorboard=push_to_tensorboard, - public=public_run, - hub_results_org=results_org, - use_wandb=wandb, - ) - - parallelism_manager = ParallelismManager.NONE - - if model_args.endswith(".yaml"): - model_config = InferenceProvidersModelConfig.from_path(model_args) - else: - model_config = InferenceProvidersModelConfig.from_args(model_args) - - pipeline_params = PipelineParameters( - launcher_type=parallelism_manager, - load_tasks_multilingual=load_tasks_multilingual, - job_id=job_id, - dataset_loading_processes=dataset_loading_processes, - custom_tasks_directory=custom_tasks, - num_fewshot_seeds=num_fewshot_seeds, - max_samples=max_samples, - load_responses_from_details_date_id=None, - remove_reasoning_tags=remove_reasoning_tags, - reasoning_tags=reasoning_tags, - ) - pipeline = Pipeline( - tasks=tasks, - pipeline_parameters=pipeline_params, - evaluation_tracker=evaluation_tracker, - model_config=model_config, - ) - - pipeline.evaluate() - - pipeline.show_results() - - results = pipeline.get_results() - - pipeline.save_and_push_results() - - return results diff --git a/src/lighteval/main_inspect.py b/src/lighteval/main_inspect.py deleted file mode 100644 index 4aa5cbf87..000000000 --- a/src/lighteval/main_inspect.py +++ /dev/null @@ -1,568 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import logging -from collections import defaultdict -from datetime import datetime -from typing import Literal - -import requests -from huggingface_hub import HfApi -from inspect_ai import Epochs, Task, task -from inspect_ai import eval_set as inspect_ai_eval_set -from inspect_ai.dataset import hf_dataset -from inspect_ai.log import bundle_log_dir -from inspect_ai.scorer import exact -from inspect_ai.solver import generate, system_message -from pytablewriter import MarkdownTableWriter -from typer import Argument, Option -from typing_extensions import Annotated - -from lighteval.models.abstract_model import InspectAIModelConfig -from lighteval.tasks.lighteval_task import LightevalTaskConfig - - -logger = logging.getLogger(__name__) - - -@task -def get_inspect_ai_task( - lighteval_task_config: LightevalTaskConfig, - epochs: int = 1, - epochs_reducer: Literal["mean", "median", "mode", "max", "at_least_{n}", "pass_at_{k}"] | None = None, -) -> Task: - name = lighteval_task_config.name - sample_fields = lighteval_task_config.sample_fields - - if sample_fields is None: - raise ValueError( - f"Task {name} is not supported by inspect_ai yet. You can either define it or use a different backend, `lighteval --help`" - ) - - dataset_repo = lighteval_task_config.hf_repo - dataset_subset = lighteval_task_config.hf_subset - dataset_split = lighteval_task_config.evaluation_splits[0] - - dataset = hf_dataset(dataset_repo, name=dataset_subset, split=dataset_split, sample_fields=sample_fields) - if lighteval_task_config.filter is not None: - dataset = dataset.filter(lighteval_task_config.filter) - solver = lighteval_task_config.solver or [ - generate(cache=True), - ] - scorers = lighteval_task_config.scorer or exact() - # TODO: have per task epoch and epoch reducer - - if lighteval_task_config.num_fewshots > 0: - name += f"_{lighteval_task_config.num_fewshots}_shots" - # TODO: use fewshot split - fewshots = hf_dataset( - path=dataset_repo, - name=dataset_subset, - split=dataset_split, - sample_fields=sample_fields, - shuffle=True, - seed=42, - limit=lighteval_task_config.num_fewshots, - ) - solver.insert( - 0, - system_message("\n\n".join([lighteval_task_config.sample_to_fewshot(sample) for sample in fewshots])), - ) - - return Task(dataset=dataset, solver=solver, scorer=scorers, name=name, epochs=Epochs(epochs, epochs_reducer)) - - -def push_to_hub(bundle_dir: str, repo_id: str, public: bool = False): - api = HfApi() - api.create_repo(repo_id=repo_id, repo_type="space", space_sdk="static", exist_ok=True, private=not public) - api.upload_folder(repo_id=repo_id, repo_type="space", folder_path=bundle_dir) - print(f"Details pushed to https://huggingface.co/spaces/{repo_id}") - - -def mean_metrics_by_prefix(results_per_model_per_task, sep=":"): - out = {} - for model, tasks in results_per_model_per_task.items(): - pref_metrics = defaultdict(lambda: defaultdict(list)) - # Collect both per-task metrics and values for prefix aggregation - per_model_out = {} - for task_name, metrics in tasks.items(): - if sep not in task_name: - # No subtasks: keep metrics as-is for this task - per_task_vals = {} - for mname, metric in metrics.items(): - per_task_vals[mname] = getattr(metric, "value", metric) - per_model_out[task_name] = per_task_vals - continue - prefix = task_name.split(sep, 1)[0] - # Keep non-averaged task metrics - per_task_vals = {} - for mname, metric in metrics.items(): - value = getattr(metric, "value", metric) - per_task_vals[mname] = value - pref_metrics[prefix][mname].append(value) - per_model_out[task_name] = per_task_vals - # Add the averaged metrics per prefix - for p, md in pref_metrics.items(): - per_model_out[p] = {m: sum(v) / len(v) for m, v in md.items()} - out[model] = per_model_out - return out - - -def results_to_markdown_table( - results_per_model_per_task, - metric: str = "accuracy", - stderr_metric: str = "stderr", - max_total_columns: int | None = None, - means_only_task_threshold: int = 10, -) -> str: - cols = _collect_columns(results_per_model_per_task, means_only_task_threshold, max_total_columns) - - writer = MarkdownTableWriter() - writer.headers = ["Model"] + cols - - rows = [] - for model in sorted(results_per_model_per_task.keys()): - row = [model] - data = results_per_model_per_task[model] - for col in cols: - row.append(_format_metric_cell(data, col, metric, stderr_metric)) - rows.append(row) - - writer.value_matrix = rows - return writer.dumps() - - -def _collect_columns( - results_per_model_per_task, means_only_task_threshold: int, max_total_columns: int | None -) -> list[str]: - all_cols = set() - for model_data in results_per_model_per_task.values(): - all_cols.update(model_data.keys()) - agg_cols = sorted([c for c in all_cols if ":" not in c]) - task_cols = sorted([c for c in all_cols if ":" in c]) - - if len(task_cols) > means_only_task_threshold: - logger.info( - f"Only showing the meaned tasks (aggregates only) because there are more than {means_only_task_threshold} tasks" - ) - return agg_cols - - cols = agg_cols + task_cols - if max_total_columns is not None and len(cols) > max_total_columns: - keep_left = max(1, max_total_columns // 2) - keep_right = max_total_columns - keep_left - left_cols = cols[:keep_left] - right_cols = cols[-keep_right:] if keep_right > 0 else [] - return left_cols + ["…"] + right_cols - return cols - - -def _format_metric_cell(data: dict, col: str, metric: str, stderr_metric: str) -> str: - if col == "…": - return "…" - metrics = data.get(col) - if not metrics: - return "-" - val = metrics.get(metric) - if isinstance(val, dict): - val = val.get("value", None) - if val is not None: - return "%.2f" % val - return "-" - - -def _get_huggingface_providers(model_id: str): - model_id = model_id.replace("hf-inference-providers/", "").replace(":all", "") - url = f"https://huggingface.co/api/models/{model_id}" - params = {"expand[]": "inferenceProviderMapping"} - response = requests.get(url, params=params) - response.raise_for_status() # raise exception for HTTP errors - data = response.json() - # Extract provider mapping if available - providers = data.get("inferenceProviderMapping", {}) - - live_providers = [] - for provider, info in providers.items(): - if info.get("status") == "live": - live_providers.append(provider) - - return live_providers - - -HELP_PANEL_NAME_1 = "Modeling Parameters" -HELP_PANEL_NAME_2 = "Task Parameters" -HELP_PANEL_NAME_3 = "Connection and parallelization parameters" -HELP_PANEL_NAME_4 = "Logging parameters" - - -def eval( # noqa C901 - models: Annotated[list[str], Argument(help="Models to evaluate")], - tasks: Annotated[str, Argument(help="Tasks to evaluate")], - # model arguments - model_base_url: Annotated[ - str | None, - Option( - help="Base URL for communicating with the model API.", - rich_help_panel=HELP_PANEL_NAME_1, - ), - ] = None, - model_roles: Annotated[ - str | None, - Option( - help="Model creation args (as a dictionary or as a path to a JSON or YAML config file)", - rich_help_panel=HELP_PANEL_NAME_1, - ), - ] = None, - max_tokens: Annotated[ - int | None, - Option( - help="The maximum number of tokens that can be generated in the completion (default is model specific)", - rich_help_panel=HELP_PANEL_NAME_1, - ), - ] = None, - system_message: Annotated[ - str | None, - Option( - help="System message to use, overrides the task defined system and default one.", - rich_help_panel=HELP_PANEL_NAME_1, - ), - ] = None, - temperature: Annotated[ - float | None, - Option( - help="Controls randomness in the model's output. Lower values make the model more deterministic and focused, while higher values make it more creative and varied.", - rich_help_panel=HELP_PANEL_NAME_1, - ), - ] = None, - top_p: Annotated[ - float | None, - Option( - help="An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass.", - rich_help_panel=HELP_PANEL_NAME_1, - ), - ] = None, - top_k: Annotated[ - int | None, - Option( - help="The number of highest probability vocabulary tokens to keep for each step of decoding.", - rich_help_panel=HELP_PANEL_NAME_1, - ), - ] = None, - frequence_penalty: Annotated[ - float | None, - Option( - help="Number between -2.0 and 2.0, Penalizes tokens that appear in the text too frequently, reducing repetition.", - rich_help_panel=HELP_PANEL_NAME_1, - ), - ] = None, - presence_penalty: Annotated[ - float | None, - Option( - help="Number between -2.0 and 2.0, Penalizes tokens that appear in the text, increasing diversity of the generated text.", - rich_help_panel=HELP_PANEL_NAME_1, - ), - ] = None, - logit_bias: Annotated[ - str | None, - Option( - help="Bias for each token, can be used to prioritize or deprioritize certain tokens, for example 10=100, -10=-100.", - rich_help_panel=HELP_PANEL_NAME_1, - ), - ] = None, - seed: Annotated[ - int | None, Option(help="Random seed to use for reproducibility", rich_help_panel=HELP_PANEL_NAME_1) - ] = None, - stop_seqs: Annotated[ - list[str] | None, - Option( - help="Stop sequences to use, can be used to stop the generation of the text.", - rich_help_panel=HELP_PANEL_NAME_1, - ), - ] = None, - num_choices: Annotated[ - int | None, - Option(help="The number of choices to generate for each step of decoding.", rich_help_panel=HELP_PANEL_NAME_1), - ] = None, - best_of: Annotated[ - int | None, - Option( - help="Generates best_of completions server-side and returns the one with the highest log probability per token.", - rich_help_panel=HELP_PANEL_NAME_1, - ), - ] = None, - log_probs: Annotated[ - bool | None, - Option( - help="Returns log probabilities for each token in the generated text", rich_help_panel=HELP_PANEL_NAME_1 - ), - ] = None, - top_logprobs: Annotated[ - int | None, - Option(help="How many most likely tokens to return at each forward step.", rich_help_panel=HELP_PANEL_NAME_1), - ] = None, - cache_prompt: Annotated[ - bool | None, Option(help="Cache prompt prefix.", rich_help_panel=HELP_PANEL_NAME_1) - ] = None, - reasoning_effort: Annotated[ - str | None, Option(help="Value: `minimal`, `low`, `medium`, `high`", rich_help_panel=HELP_PANEL_NAME_1) - ] = None, - reasoning_tokens: Annotated[ - int | None, - Option(help="Maximum number of tokens to generate for reasoning", rich_help_panel=HELP_PANEL_NAME_1), - ] = None, - reasoning_history: Annotated[ - bool | None, - Option( - help="values: `none`, `all`, `last`, `auto`. Include reasoning in chat message history sent to generate (defaults to “auto”, which uses the recommended default for each provider)", - rich_help_panel=HELP_PANEL_NAME_1, - ), - ] = None, - response_format: Annotated[ - str | None, Option(help="JSON schema for the response", rich_help_panel=HELP_PANEL_NAME_1) - ] = None, - parallel_tool_calls: Annotated[ - bool | None, Option(help="Enable parallel tool calls", rich_help_panel=HELP_PANEL_NAME_1) - ] = None, - max_tool_output: Annotated[ - int | None, - Option(help="Maximum number of tokens to generate for tool output", rich_help_panel=HELP_PANEL_NAME_1), - ] = None, - internal_tools: Annotated[ - bool | None, Option(help="Enable internal tools", rich_help_panel=HELP_PANEL_NAME_1) - ] = None, - model_args: Annotated[ - str | None, Option(help="Provider specific arguments: example: 'device=1'", rich_help_panel=HELP_PANEL_NAME_1) - ] = None, - # task parameters - custom_tasks: Annotated[ - str | None, - Option( - help="Path to a Python file containing custom task definitions. The file should define a TASKS_TABLE with LightevalTaskConfig objects.", - rich_help_panel=HELP_PANEL_NAME_2, - ), - ] = None, - max_samples: Annotated[ - int | None, - Option( - help="Maximum number of samples to use per task. If your task has multiple subtasks, this will be the maximum number of samples to use per subtask.", - rich_help_panel=HELP_PANEL_NAME_2, - ), - ] = None, - # Metric parameters - epochs: Annotated[ - int, - Option( - help="Number of times to evaluate the model on the task, the results will be aggregated by the specified reducer.", - rich_help_panel=HELP_PANEL_NAME_2, - ), - ] = 1, - epochs_reducer: Annotated[ - str | None, - Option( - help="Epochs Reducer to use: mean, median, mode, max, at_least_{n}, pass_at_{k}", - rich_help_panel=HELP_PANEL_NAME_2, - ), - ] = None, - # Connection and parallelization parameters - max_connections: Annotated[ - int, - Option( - help="Maximum number of concurrent connections to use for each model", rich_help_panel=HELP_PANEL_NAME_3 - ), - ] = 50, - timeout: Annotated[ - int, Option(help="Timeout in seconds for each connection", rich_help_panel=HELP_PANEL_NAME_3) - ] = 30, - retry_on_error: Annotated[ - int, Option(help="Number of times to retry on error", rich_help_panel=HELP_PANEL_NAME_3) - ] = 1, - max_retries: Annotated[ - int, Option(help="Maximum number of retries to use", rich_help_panel=HELP_PANEL_NAME_3) - ] = 5, - max_tasks: Annotated[ - int | None, Option(help="Maximum number of tasks to evaluate in parallel", rich_help_panel=HELP_PANEL_NAME_3) - ] = None, - # Logging parameters - log_dir: Annotated[ - str | None, - Option(help="Log directory to use, will be created if it doesn't exist", rich_help_panel=HELP_PANEL_NAME_4), - ] = None, - log_dir_allow_dirty: Annotated[ - bool, Option(help="Allow dirty log directory", rich_help_panel=HELP_PANEL_NAME_4) - ] = True, - display: Annotated[ - Literal["rich", "full", "conversations", "plain", "log", "none"], - Option(help="Display", rich_help_panel=HELP_PANEL_NAME_4), - ] = "rich", - bundle_dir: Annotated[ - str | None, - Option(help="Bundle directory to use, will be created if it doesn't exist", rich_help_panel=HELP_PANEL_NAME_4), - ] = None, - bundle_overwrite: Annotated[ - bool, - Option(help="Overwrite bundle directory if it exists", rich_help_panel=HELP_PANEL_NAME_4), - ] = True, - repo_id: Annotated[ - str | None, - Option(help="Repository ID to use, will be created if it doesn't exist", rich_help_panel=HELP_PANEL_NAME_4), - ] = None, - public: Annotated[ - bool, - Option( - help="Whether to make the uploaded results and details public on the Hugging Face Hub. If False, datasets will be private.", - rich_help_panel=HELP_PANEL_NAME_4, - ), - ] = False, -): - from lighteval.tasks.registry import Registry - - registry = Registry(tasks=tasks, custom_tasks=None, load_multilingual=False) - task_configs = registry.task_to_configs - inspect_ai_tasks = [] - - for task_name, task_configs in task_configs.items(): - for task_config in task_configs: - inspect_ai_tasks.append(get_inspect_ai_task(task_config, epochs=epochs, epochs_reducer=epochs_reducer)) - - if model_args is not None: - model_args = InspectAIModelConfig._parse_args(model_args) - else: - model_args = {} - - for model in models: - if model.split("/")[0] == "hf-inference-providers" and model.split(":")[-1] == "all": - providers = _get_huggingface_providers(model) - models = [f"{model.replace(':all', '')}:{provider}" for provider in providers] - - if log_dir is None: - log_dir = f"lighteval-logs-{datetime.now().strftime('%Y%m%d%H%M%S')}" - - success, logs = inspect_ai_eval_set( - inspect_ai_tasks, - model=models, - max_connections=max_connections, - timeout=timeout, - retry_on_error=retry_on_error, - max_retries=max_retries, - limit=max_samples, - max_tasks=max_tasks, - log_dir=log_dir, - log_dir_allow_dirty=log_dir_allow_dirty, - display=display, - model_args=model_args, - max_tokens=max_tokens, - system_message=system_message, - temperature=temperature, - top_p=top_p, - top_k=top_k, - frequence_penalty=frequence_penalty, - presence_penalty=presence_penalty, - seed=seed, - stop_seqs=stop_seqs, - num_choices=num_choices, - best_of=best_of, - log_probs=log_probs, - top_logprobs=top_logprobs, - cache_prompt=cache_prompt, - reasoning_effort=reasoning_effort, - reasoning_tokens=reasoning_tokens, - reasoning_history=reasoning_history, - response_format=response_format, - parallel_tool_calls=parallel_tool_calls, - max_tool_output=max_tool_output, - internal_tools=internal_tools, - bundle_dir=bundle_dir, - bundle_overwrite=bundle_overwrite, - ) - - if not success: - print("Error evaluating models") - print(f"run the same command with --log-dir {log_dir} to retry !") - return - - results_per_model_per_task = {} - - for model in models: - results_per_model_per_task[model] = {} - - for log in logs: - if log.eval.model == model: - results_per_model_per_task[model][log.eval.task] = log.results.metrics - - results_per_model_per_task_agg = mean_metrics_by_prefix(results_per_model_per_task) - table_md = results_to_markdown_table(results_per_model_per_task_agg) - - if repo_id is not None: - if bundle_dir is not None: - push_to_hub(bundle_dir, repo_id, public=public) - - print() - print(table_md) - print(f"results saved to {log_dir}") - - if log_dir is not None: - print(f'run "inspect view --log-dir {log_dir}" to view the results') - else: - print("run 'inspect view' to view the results") - - -def bundle(log_dir: str, output_dir: str, overwrite: bool = True, repo_id: str | None = None, public: bool = False): - bundle_log_dir(log_dir=log_dir, output_dir=output_dir, overwrite=overwrite) - - if repo_id is not None: - push_to_hub(output_dir, repo_id, public=public) - - -if __name__ == "__main__": - tasks = [ - "gsm8k", - "agieval", - "hle", - "ifeval", - "gpqa", - "ifbench_test", - "mmlu_pro", - "mixeval", - "aimo", - "anli", - "arc", - "arithmetic", - "asdiv", - "babi_qa", - "bbq", - "bigbench", - "bigbench_hard", - "blimp", - "bold", - "boolq", - "civil_comments", - "commonsenseqa", - "covid_dialog", - "dyck_language", - "math_500", - "musr", - "olympiad_bench", - "simpleqa", - "tiny_benchmarks", - ] - model = "hf-inference-providers/meta-llama/Llama-3.1-8B-Instruct:nebius" - eval(models=[model], tasks=tasks[0]) diff --git a/src/lighteval/main_nanotron.py b/src/lighteval/main_nanotron.py deleted file mode 100644 index 9399e82cd..000000000 --- a/src/lighteval/main_nanotron.py +++ /dev/null @@ -1,121 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -# flake8: noqa: C901 -import os - -import yaml -from typer import Option -from typing_extensions import Annotated -from yaml import SafeLoader - -from lighteval.cli_args import ( - load_tasks_multilingual, - reasoning_tags, - remove_reasoning_tags, -) -from lighteval.utils.imports import requires - - -SEED = 1234 - - -@requires("nanotron") -def nanotron( - checkpoint_config_path: Annotated[ - str, Option(help="Path to the nanotron checkpoint YAML or python config file, potentially on s3.") - ], - lighteval_config_path: Annotated[str, Option(help="Path to a YAML config to be used for the evaluation.")], - load_tasks_multilingual: load_tasks_multilingual.type = load_tasks_multilingual.default, - remove_reasoning_tags: remove_reasoning_tags.type = remove_reasoning_tags.default, - reasoning_tags: reasoning_tags.type = reasoning_tags.default, -): - """ - Evaluate models using nanotron as backend. - """ - from nanotron.config import GeneralArgs, ModelArgs, TokenizerArgs, get_config_from_dict, get_config_from_file - - from lighteval.logging.evaluation_tracker import EvaluationTracker - from lighteval.models.nanotron import ( - FullNanotronConfig, - LightEvalConfig, - ) - from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters - - # Create nanotron config - if not checkpoint_config_path.endswith(".yaml"): - raise ValueError("The checkpoint path should point to a YAML file") - - with open(checkpoint_config_path) as f: - nanotron_yaml = yaml.load(f, Loader=SafeLoader) - - model_config, tokenizer_config, general_config = [ - get_config_from_dict( - nanotron_yaml[key], - config_class=config_class, - skip_unused_config_keys=True, - skip_null_keys=True, - ) - for key, config_class in [("model", ModelArgs), ("tokenizer", TokenizerArgs), ("general", GeneralArgs)] - ] - - # Load lighteval config - lighteval_config: LightEvalConfig = get_config_from_file(lighteval_config_path, config_class=LightEvalConfig) # type: ignore - nanotron_config = FullNanotronConfig(lighteval_config, model_config, tokenizer_config, general_config) - - evaluation_tracker = EvaluationTracker( - output_dir=lighteval_config.logging.output_dir, - results_path_template=lighteval_config.logging.results_path_template, - hub_results_org=lighteval_config.logging.results_org, - public=lighteval_config.logging.public_run, - push_to_hub=lighteval_config.logging.push_to_hub, - push_to_tensorboard=lighteval_config.logging.push_to_tensorboard, - save_details=lighteval_config.logging.save_details, - tensorboard_metric_prefix=lighteval_config.logging.tensorboard_metric_prefix, - nanotron_run_info=nanotron_config.nanotron_general, - ) - - pipeline_parameters = PipelineParameters( - launcher_type=ParallelismManager.NANOTRON, - job_id=os.environ.get("SLURM_JOB_ID", 0), - nanotron_checkpoint_path=checkpoint_config_path, - dataset_loading_processes=lighteval_config.tasks.dataset_loading_processes, - custom_tasks_directory=lighteval_config.tasks.custom_tasks, - num_fewshot_seeds=1, - max_samples=lighteval_config.tasks.max_samples, - remove_reasoning_tags=remove_reasoning_tags, - reasoning_tags=reasoning_tags, - load_tasks_multilingual=load_tasks_multilingual, - ) - - pipeline = Pipeline( - tasks=lighteval_config.tasks.tasks, - pipeline_parameters=pipeline_parameters, - evaluation_tracker=evaluation_tracker, - model_config=nanotron_config, - ) - - pipeline.evaluate() - - pipeline.show_results() - - pipeline.save_and_push_results() diff --git a/src/lighteval/main_sglang.py b/src/lighteval/main_sglang.py deleted file mode 100644 index ab86349f9..000000000 --- a/src/lighteval/main_sglang.py +++ /dev/null @@ -1,130 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The SGLang Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from lighteval.cli_args import ( - custom_tasks, - dataset_loading_processes, - job_id, - load_responses_from_details_date_id, - load_tasks_multilingual, - max_samples, - model_args, - num_fewshot_seeds, - output_dir, - public_run, - push_to_hub, - push_to_tensorboard, - reasoning_tags, - remove_reasoning_tags, - results_org, - results_path_template, - save_details, - tasks, - wandb, -) - - -def sglang( - # === general === - model_args: model_args.type, - tasks: tasks.type, - # === Common parameters === - load_tasks_multilingual: load_tasks_multilingual.type = load_tasks_multilingual.default, - dataset_loading_processes: dataset_loading_processes.type = dataset_loading_processes.default, - custom_tasks: custom_tasks.type = custom_tasks.default, - num_fewshot_seeds: num_fewshot_seeds.type = num_fewshot_seeds.default, - load_responses_from_details_date_id: load_responses_from_details_date_id.type = load_responses_from_details_date_id.default, - remove_reasoning_tags: remove_reasoning_tags.type = remove_reasoning_tags.default, - reasoning_tags: reasoning_tags.type = reasoning_tags.default, - # === saving === - output_dir: output_dir.type = output_dir.default, - results_path_template: results_path_template.type = results_path_template.default, - push_to_hub: push_to_hub.type = push_to_hub.default, - push_to_tensorboard: push_to_tensorboard.type = push_to_tensorboard.default, - public_run: public_run.type = public_run.default, - results_org: results_org.type = results_org.default, - save_details: save_details.type = save_details.default, - wandb: wandb.type = wandb.default, - # === debug === - max_samples: max_samples.type = max_samples.default, - job_id: job_id.type = job_id.default, -): - """Evaluate models using sglang as backend. - - Returns: - dict: Evaluation results containing metrics and scores for all tasks - """ - import yaml - - from lighteval.logging.evaluation_tracker import EvaluationTracker - from lighteval.models.sglang.sglang_model import SGLangModelConfig - from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters - - evaluation_tracker = EvaluationTracker( - output_dir=output_dir, - results_path_template=results_path_template, - save_details=save_details, - push_to_hub=push_to_hub, - push_to_tensorboard=push_to_tensorboard, - public=public_run, - hub_results_org=results_org, - use_wandb=wandb, - ) - - pipeline_params = PipelineParameters( - launcher_type=ParallelismManager.SGLANG, - job_id=job_id, - load_tasks_multilingual=load_tasks_multilingual, - dataset_loading_processes=dataset_loading_processes, - custom_tasks_directory=custom_tasks, - num_fewshot_seeds=num_fewshot_seeds, - max_samples=max_samples, - load_responses_from_details_date_id=load_responses_from_details_date_id, - remove_reasoning_tags=remove_reasoning_tags, - reasoning_tags=reasoning_tags, - ) - - if model_args.endswith(".yaml"): - with open(model_args, "r") as f: - metric_options = yaml.safe_load(f).get("metric_options", {}) - model_config = SGLangModelConfig.from_path(model_args) - else: - metric_options = {} - model_config = SGLangModelConfig.from_args(model_args) - - pipeline = Pipeline( - tasks=tasks, - pipeline_parameters=pipeline_params, - evaluation_tracker=evaluation_tracker, - model_config=model_config, - metric_options=metric_options, - ) - - pipeline.evaluate() - - pipeline.show_results() - - results = pipeline.get_results() - - pipeline.save_and_push_results() - - return results diff --git a/src/lighteval/main_tasks.py b/src/lighteval/main_tasks.py deleted file mode 100644 index fb71b1b77..000000000 --- a/src/lighteval/main_tasks.py +++ /dev/null @@ -1,109 +0,0 @@ -# MIT License - -# Copyright (c) 2024 Taratra D. RAHARISON and The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -import json -import logging - -import typer -from typer import Argument, Option -from typing_extensions import Annotated - -from lighteval.cli_args import custom_tasks, load_tasks_multilingual - - -app = typer.Typer() - - -@app.command() -def inspect( - tasks: Annotated[str, Argument(help="Id of tasks or path to a text file with a list of tasks")], - load_multilingual: Annotated[bool, Option(help="Whether to load multilingual tasks")] = False, - custom_tasks: custom_tasks.type = custom_tasks.default, - num_samples: Annotated[int, Option(help="Number of samples to display")] = 10, - show_config: Annotated[bool, Option(help="Will display the full task config")] = False, -): - """Inspect a tasks""" - from dataclasses import asdict - from pprint import pformat - - from rich import print - - from lighteval.tasks.registry import Registry - - registry = Registry(tasks=tasks, custom_tasks=custom_tasks, load_multilingual=load_multilingual) - - # Loading task - task_dict = registry.load_tasks() - for name, task in task_dict.items(): - print("-" * 10, name, "-" * 10) - if show_config: - print("-" * 10, "CONFIG") - task.config.print() - for ix, sample in enumerate(task.eval_docs()[: int(num_samples)]): - if ix == 0: - print("-" * 10, "SAMPLES") - print(f"-- sample {ix} --") - print(pformat(asdict(sample), indent=2)) - - -@app.command() -def list( - load_tasks_multilingual: load_tasks_multilingual.type = load_tasks_multilingual.default, - custom_tasks: custom_tasks.type = custom_tasks.default, -): - """List all tasks""" - from lighteval.tasks.registry import Registry - - registry = Registry(custom_tasks=custom_tasks, load_multilingual=load_tasks_multilingual) - registry.print_all_tasks() - - -@app.command() -def create(template: str, task_name: str, dataset_name: str): - """Create a new task""" - logger = logging.getLogger(__name__) - - logger.info(f"Creating task for dataset {dataset_name}") - - with open(template, "r") as f: - content = f.read() - - content = content.replace("HF_TASK_NAME", task_name) - content = content.replace("HF_DATASET_NAME", dataset_name) - - with open(f"custom_{task_name}_task.py", "w+") as f: - f.write(content) - - logger.info(f"Task created in custom_{task_name}_task.py") - - -@app.command() -def dump( - load_tasks_multilingual: load_tasks_multilingual.type = load_tasks_multilingual.default, - custom_tasks: custom_tasks.type = custom_tasks.default, -): - """Dump all task names, metadata, and docstrings as JSON""" - from lighteval.tasks.registry import Registry - - registry = Registry(custom_tasks=custom_tasks, load_multilingual=load_tasks_multilingual) - modules_data = registry.get_tasks_dump() - - print(json.dumps(modules_data, indent=2, default=str)) diff --git a/src/lighteval/main_vllm.py b/src/lighteval/main_vllm.py deleted file mode 100644 index fcfb9652a..000000000 --- a/src/lighteval/main_vllm.py +++ /dev/null @@ -1,138 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from typing import Optional - -from typer import Option -from typing_extensions import Annotated - -from lighteval.cli_args import ( - HELP_PANEL_NAME_4, - custom_tasks, - dataset_loading_processes, - job_id, - load_responses_from_details_date_id, - max_samples, - model_args, - num_fewshot_seeds, - output_dir, - public_run, - push_to_hub, - push_to_tensorboard, - reasoning_tags, - remove_reasoning_tags, - results_org, - results_path_template, - save_details, - tasks, - wandb, -) - - -def vllm( - # === general === - model_args: model_args.type, - tasks: tasks.type, - # === Common parameters === - cot_prompt: Annotated[ - Optional[str], Option(help="Use chain of thought prompt for evaluation.", rich_help_panel=HELP_PANEL_NAME_4) - ] = None, - dataset_loading_processes: dataset_loading_processes.type = dataset_loading_processes.default, - custom_tasks: custom_tasks.type = custom_tasks.default, - num_fewshot_seeds: num_fewshot_seeds.type = num_fewshot_seeds.default, - load_responses_from_details_date_id: load_responses_from_details_date_id.type = load_responses_from_details_date_id.default, - remove_reasoning_tags: remove_reasoning_tags.type = remove_reasoning_tags.default, - reasoning_tags: reasoning_tags.type = reasoning_tags.default, - # === saving === - output_dir: output_dir.type = output_dir.default, - results_path_template: results_path_template.type = results_path_template.default, - push_to_hub: push_to_hub.type = push_to_hub.default, - push_to_tensorboard: push_to_tensorboard.type = push_to_tensorboard.default, - public_run: public_run.type = public_run.default, - results_org: results_org.type = results_org.default, - save_details: save_details.type = save_details.default, - wandb: wandb.type = wandb.default, - # === debug === - max_samples: max_samples.type = max_samples.default, - job_id: job_id.type = job_id.default, -): - """Evaluate models using vllm as backend. - - Returns: - dict: Evaluation results containing metrics and scores for all tasks - """ - import yaml - - from lighteval.logging.evaluation_tracker import EvaluationTracker - from lighteval.models.vllm.vllm_model import VLLMModelConfig - from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters - - evaluation_tracker = EvaluationTracker( - output_dir=output_dir, - results_path_template=results_path_template, - save_details=save_details, - push_to_hub=push_to_hub, - push_to_tensorboard=push_to_tensorboard, - public=public_run, - hub_results_org=results_org, - use_wandb=wandb, - ) - - pipeline_params = PipelineParameters( - launcher_type=ParallelismManager.VLLM, - job_id=job_id, - dataset_loading_processes=dataset_loading_processes, - custom_tasks_directory=custom_tasks, - num_fewshot_seeds=num_fewshot_seeds, - max_samples=max_samples, - cot_prompt=cot_prompt, - load_responses_from_details_date_id=load_responses_from_details_date_id, - remove_reasoning_tags=remove_reasoning_tags, - reasoning_tags=reasoning_tags, - ) - - if model_args.endswith(".yaml"): - with open(model_args, "r") as f: - metric_options = yaml.safe_load(f).get("metric_options", {}) - model_config = VLLMModelConfig.from_path(model_args) - else: - metric_options = {} - model_config = VLLMModelConfig.from_args(model_args) - - pipeline = Pipeline( - tasks=tasks, - pipeline_parameters=pipeline_params, - evaluation_tracker=evaluation_tracker, - model_config=model_config, - metric_options=metric_options, - ) - - pipeline.evaluate() - - pipeline.show_results() - - results = pipeline.get_results() - details = pipeline.get_details() - - pipeline.save_and_push_results() - - return results, details diff --git a/src/lighteval/metrics/__init__.py b/src/lighteval/metrics/__init__.py deleted file mode 100644 index d61b13764..000000000 --- a/src/lighteval/metrics/__init__.py +++ /dev/null @@ -1,62 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -from lighteval.metrics.utils.metric_utils import Metric -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.requests import Doc - - -def apply_metric(responses: list[ModelResponse], docs: list[Doc], metrics: list[Metric]): - # Separate batched and non-batched metrics - batched_metrics = [m for m in metrics if m.batched_compute] - non_batched_metrics = [m for m in metrics if not m.batched_compute] - - outputs = [] - - # Handle batched metrics first - batched_outputs = [] - if batched_metrics: - for metric in batched_metrics: - metric_outputs = metric.compute_sample(responses=responses, docs=docs) - batched_outputs.append(metric_outputs) - - # Initialize outputs with the correct structure - for i in range(len(docs)): - output = {} - - # Add batched metric results for this sample - for metric, metric_outputs in zip(batched_metrics, batched_outputs): - output.update({metric.metric_name: metric_outputs[metric.metric_name][i]}) - - # Add non-batched metric results for this sample - for metric in non_batched_metrics: - output.update( - metric.compute_sample( - model_response=responses[i], - doc=docs[i], - ) - ) - - outputs.append(output) - - return outputs diff --git a/src/lighteval/metrics/dynamic_metrics.py b/src/lighteval/metrics/dynamic_metrics.py deleted file mode 100644 index 66ed91c3a..000000000 --- a/src/lighteval/metrics/dynamic_metrics.py +++ /dev/null @@ -1,268 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import logging -from typing import Callable, Literal, Sequence - -import numpy as np - -from lighteval.metrics.metrics_sample import ( - ExactMatches, - F1_score, - LoglikelihoodAcc, - NormalizedMultiChoiceProbability, - Probability, -) -from lighteval.metrics.normalizations import ( - LogProbNormalization, - LogProbTokenNorm, - get_multilingual_normalizer, -) -from lighteval.metrics.utils.extractive_match_utils import ( # noqa: F401 - ExprExtractionConfig, - ExtractionTarget, - LatexExtractionConfig, - extract_target_from_pred, - get_extraction_regexes, -) -from lighteval.metrics.utils.math_comparison import compare_gold_target -from lighteval.metrics.utils.metric_utils import SampleLevelComputation, SampleLevelMetric -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.requests import Doc, SamplingMethod -from lighteval.utils.language import Language -from lighteval.utils.timeout import timeout - - -logger = logging.getLogger(__name__) - - -class LogLikelihoodAccMetric(SampleLevelMetric): - def __init__(self, normalization: LogProbNormalization | None = None): - """Creates an accuracy (loglikelihood) metric, which returns accuracy given normalization.""" - super().__init__( - metric_name="acc" + (f"_{normalization.name}" if normalization else ""), - sample_level_fn=LoglikelihoodAcc(logprob_normalization=normalization), - category=SamplingMethod.LOGPROBS, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - - -class NormalizedMultiChoiceProbMetric(SampleLevelMetric): - def __init__( - self, - normalization: LogProbNormalization | None = None, - aggregation_function: Callable[[np.ndarray], float] = np.max, - ): - """Creates a normalized multi-choice probability metric, which returns the probability of the gold choice / sum of probabilities of all choices (after logprobs are normalized).""" - super().__init__( - metric_name="normalized_mc_prob" + (f"_{normalization.name}" if normalization else ""), - sample_level_fn=NormalizedMultiChoiceProbability( - log_prob_normalization=normalization, aggregation_function=aggregation_function - ), - category=SamplingMethod.LOGPROBS, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - - -class ProbabilityMetric(SampleLevelMetric): - def __init__( - self, - normalization: LogProbTokenNorm | None = None, - aggregation_function: Callable[[np.ndarray], float] = np.max, - ): - """Creates a probability metric, which returns the probability of the gold choice given normalization.""" - super().__init__( - metric_name="prob" + (f"_{normalization.name}" if normalization else ""), - sample_level_fn=Probability(normalization=normalization, aggregation_function=aggregation_function), - category=SamplingMethod.LOGPROBS, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - - -class MultilingualQuasiF1ScoreMetric(SampleLevelMetric): - def __init__(self, language: Language, aggregation_function: Callable[[list[float]], float] = max): - """Creates a language-aware F1 score metric, which returns the F1 score. - - Args: - language: The language of the samples. - aggregation_function: Aggregation samples to use when multiple golds are present. - """ - super().__init__( - metric_name=f"f1_{language.value}", - sample_level_fn=F1_score( - normalize_gold=get_multilingual_normalizer(language), - normalize_pred=get_multilingual_normalizer(language), - aggregation_function=aggregation_function, - ), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - - -class MultilingualQuasiExactMatchMetric(SampleLevelMetric): - def __init__( - self, - language: Language, - match_type: Literal["prefix", "suffix", "full"] = "full", - aggregation_function: Callable[[list[float]], float] = max, - ): - """Creates a language-aware exact match metric, which returns the exact match score - Args: - language: The language of the samples. - match_type: The type of match to use - - "prefix": Prefixes must match - - "suffix": Suffixes must match - - "full": Full strings must match - aggregation_function: Aggregation samples to use when multiple golds are present. - """ - super().__init__( - metric_name=f"exact_match_{language.value}_{match_type}", - sample_level_fn=ExactMatches( - normalize_gold=get_multilingual_normalizer(language), - normalize_pred=get_multilingual_normalizer(language), - aggregation_function=aggregation_function, - type_exact_match=match_type, - ), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - - -class MultilingualExtractiveMatchMetric(SampleLevelComputation): - def __init__( - self, - language: Language = Language.ENGLISH, - gold_extraction_target: Sequence[ExtractionTarget] = (ExprExtractionConfig(),), - pred_extraction_target: Sequence[ExtractionTarget] = (ExprExtractionConfig(), LatexExtractionConfig()), - aggregation_function: Callable[[list[float]], float] = max, - fallback_mode: Literal["no_fallback", "first_match"] = "first_match", - extraction_mode: Literal["first_match", "any_match"] = "any_match", - precision: int = 6, - timeout_seconds: int = 5, - ): - """Creates a language-aware extractive match metric that extracts answers from the model's output. - - Known issues: - - If the task is to simplify an expression, the metric might overestimate the accuracy. This is because if the model doesn't output any anchor for the extraction (e.g final answer is..), - it's possible that the extracted prediction will be the expression to simplify. Because we do simplifications ourselves, it can thus happen that sympy will correctly simplify the expression, - thus it will match gold, despite model not doing anything. PRs to fix this are welcome. - - - There is currently no StringExtractionConfig, so if the gold is \boxed{\text{Friday}} and model outputs Friday it will not match, because nothing will be extracted. - - Args: - language: Language - The language of the samples. - gold_extraction_target: Sequence[ExtractionTarget] - Extraction targets to use for gold answers. Defaults to extracting simple math expressions. - pred_extraction_target: Sequence[ExtractionTarget] - Extraction targets to use for predictions. Defaults to extracting simple math expressions. - aggregation_function: Callable[[list[float]], float] - Function to aggregate scores when multiple golds/predictions are present. Defaults to max. - fallback_mode: Literal["no_fallback", "first_match"] - How to perform extraction. Defaults to "first_match". - - "no_fallback": Only use first successfully parsed matches - - "first_match": Use the first successfully parsed match + first match irregardless the parsing success - extraction_mode: Literal["first_match", "any_match"] - - "first_match": Only tries to extract the first regex match if it fails no other matches are tried - - "any_match": Tries to extract any regex match - - precision: int - Number of decimal places to use when comparing numerical values. Defaults to 6. - timeout_seconds: int - Timeout for the extraction (each attempt) and comparison. Defaults to 5. - - """ - self.language = language - self.gold_extraction_target = gold_extraction_target - self.pred_extraction_target = pred_extraction_target - self.aggregation_function = aggregation_function - self.fallback_mode = fallback_mode - self.extraction_mode = extraction_mode - self.precision = precision - self.timeout_seconds = timeout_seconds - - @timeout(2) - def add_to_specifics_with_timeout( - self, formatted_doc: Doc, extracted_predictions: list[list[str]], extracted_golds: list[list[str]] - ) -> None: - if formatted_doc.specific is None: - formatted_doc.specific = {} - - formatted_doc.specific["extracted_predictions"] = [ - str(pred) for preds in extracted_predictions for pred in preds - ] - formatted_doc.specific["extracted_golds"] = [str(gold) for golds in extracted_golds for gold in golds] - - def compute(self, doc: Doc, model_response: ModelResponse) -> float: - golds = doc.get_golds() - predictions = model_response.final_text - - gold_extraction_regexes = get_extraction_regexes(doc, self.gold_extraction_target, self.language) - pred_extraction_regexes = get_extraction_regexes(doc, self.pred_extraction_target, self.language) - - extracted_predictions = [ - extract_target_from_pred( - pred, pred_extraction_regexes, self.fallback_mode, self.extraction_mode, self.timeout_seconds - ) - for pred in predictions - ] - extracted_golds = [ - extract_target_from_pred( - gold, gold_extraction_regexes, self.fallback_mode, self.extraction_mode, self.timeout_seconds - ) - for gold in golds - ] - - # Assert on empty gold and warn on empty pred - if any(len(g) == 0 for g in extracted_golds): - logger.warning(f"We did not manage to extract a gold in the correct format. Gold: {golds}") - extracted_golds = [[gold] for gold in golds] - - if all(len(p) == 0 for p in extracted_predictions): - logger.warning( - f"We did not manage to extract a prediction in the correct format. Gold: {golds}, Pred: {predictions}" - ) - - # We have to use timeout because the sypmy to str conversion can be very slow - try: - self.add_to_specifics_with_timeout(doc, extracted_predictions, extracted_golds) - except TimeoutError: # noqa: E722 - logger.warning("Timeout when adding extracted predictions and golds to specific") - - return self.aggregation_function( - [ - ( - 1.0 - if any( - compare_gold_target(gold, pred, self.precision, timeout_seconds=self.timeout_seconds) - for gold in extracted_golds - ) - else 0.0 - ) - for pred in extracted_predictions - ] - ) diff --git a/src/lighteval/metrics/harness_compatibility/drop.py b/src/lighteval/metrics/harness_compatibility/drop.py deleted file mode 100644 index 93ff0028b..000000000 --- a/src/lighteval/metrics/harness_compatibility/drop.py +++ /dev/null @@ -1,183 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import re -import string -from typing import List, Set, Tuple - -import numpy as np -from scipy.optimize import linear_sum_assignment - -from lighteval.metrics.metrics_sample import SampleLevelComputation -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.requests import Doc - - -class DropMetrics(SampleLevelComputation): - def compute(self, doc: Doc, model_response: ModelResponse): # noqa: C901 - """F1 score from bag of words: comes from Harness Drop. DROP offers two metrics, - a quasi exact match and a numeracy-focused F1 score. Quasi in the sense that it - does some normalizations before matching and numeracy-focused in the sense that - if there's number mismatch between the target and prediction F1 score is set to 0. - F1 score is computed using the intersection of target and prediction's BoW - representations with the additional spice that if the answer and/or prediction is - comprised of multiple spans, a greedy matching is done between the two sets of spans - (based on the very BoW overlap) and the average over F1 of pairs is returned. - DROP also accepts multiple answers in which case, the maximum of F1/ Exact Match - between prediction and the different answers is taken. - - For more information, please refer to the section 5 of the DROP paper (https://aclanthology.org/N19-1246/). - - Todo: this code is really hard to follow, simplify when possible - - Returns: - dict: Dictionary containing 'em' (exact match) and 'f1' (F1 score) metrics - """ - max_em = 0 - max_f1 = 0 - for gold_answer in doc.specific["golds_no_preprocessing"]: - exact_match, f1_score = self._get_metrics(model_response.text, gold_answer) - if isinstance(gold_answer, list): - gold_answer = gold_answer[0] - if gold_answer.strip(): - max_em = max(max_em, exact_match) - max_f1 = max(max_f1, f1_score) - return {"em": max_em, "f1": max_f1} - - def _answer_to_bags(self, answer: List[str]) -> Tuple[List[str], List[Set[str]]]: - if isinstance(answer, (list, tuple)): - raw_spans = answer - else: - raw_spans = [answer] - normalized_spans = [] - token_bags = [] - for raw_span in raw_spans: - normalized_span = self._normalize(raw_span) - normalized_spans.append(normalized_span) - token_bags.append(set(normalized_span.split())) - return normalized_spans, token_bags - - def _get_metrics(self, predicted: List[str], gold: List[str]): - """Takes a predicted answer and a gold answer (that are both either a string or a list of - strings), and returns exact match and the DROP F1 metric for the prediction. If you are - writing a script for evaluating objects in memory (say, the output of predictions during - validation, or while training), this is the function you want to call, after using - :func:`answer_json_to_strings` when reading the gold answer from the released data file. - - Returns: - tuple: A tuple containing (exact_match, f1_score) where exact_match is a float and f1_score is a float - """ - pred_normalized_spans, pred_bags = self._answer_to_bags(predicted) - gold_normalized_spans, gold_bags = self._answer_to_bags(gold) - - if set(pred_normalized_spans) == set(gold_normalized_spans) and len(gold_normalized_spans) == len( - gold_normalized_spans - ): - exact_match = 1.0 - else: - exact_match = 0.0 - - f1_per_bag = self._align_bags(pred_bags, gold_bags) - f1 = np.mean(f1_per_bag) - f1 = round(f1, 2) - return exact_match, f1 - - def _is_number(self, text): - try: - float(text) - return True - except ValueError: - return False - - def _match_numbers_if_present(self, gold_bag: Set[str], predicted_bag: Set[str]): - gold_numbers = set() - predicted_numbers = set() - for word in gold_bag: - if self._is_number(word): - gold_numbers.add(word) - for word in predicted_bag: - if self._is_number(word): - predicted_numbers.add(word) - if (not gold_numbers) or gold_numbers.intersection(predicted_numbers): - return True - return False - - def _align_bags(self, predicted: List[Set[str]], gold: List[Set[str]]) -> np.array: - """Takes gold and predicted answer sets and first finds the optimal 1-1 alignment - between them and gets maximum metric values over all the answers. - - Returns: - np.array: Array of maximum F1 scores for each aligned bag pair - """ - scores = np.zeros([len(gold), len(predicted)]) - for gold_index, gold_item in enumerate(gold): - for pred_index, pred_item in enumerate(predicted): - if self._match_numbers_if_present(gold_item, pred_item): - scores[gold_index, pred_index] = self._compute_f1(pred_item, gold_item) - row_ind, col_ind = linear_sum_assignment(-scores) - - max_scores = np.zeros([max(len(gold), len(predicted))]) - for row, column in zip(row_ind, col_ind): - max_scores[row] = max(max_scores[row], scores[row, column]) - return max_scores - - def _compute_f1(self, predicted_bag, gold_bag): - intersection = len(gold_bag.intersection(predicted_bag)) - if not predicted_bag: - precision = 1.0 - else: - precision = intersection / float(len(predicted_bag)) - if not gold_bag: - recall = 1.0 - else: - recall = intersection / float(len(gold_bag)) - if precision == 0.0 and recall == 0.0: - return 0 - return (2 * precision * recall) / (precision + recall) - - def _remove_articles(self, text): - return re.compile(r"\b(a|an|the)\b", re.UNICODE).sub(" ", text) - - def _white_space_fix(self, text): - return " ".join(text.split()) - - def _remove_punc(self, text): - exclude = set(string.punctuation) - if not self._is_number(text): - return "".join(ch for ch in text if ch not in exclude) - else: - return text - - def _fix_number(self, text): - return str(float(text)) if self._is_number(text) else text - - def _tokenize(self, text): - return re.split(" |-", text) - - def _normalize(self, answer: str): - tokens = [ - self._white_space_fix(self._remove_articles(self._fix_number(self._remove_punc(token.lower())))) - for token in self._tokenize(answer) - ] - tokens = [token for token in tokens if token.strip()] - normalized = " ".join(tokens).strip() - return normalized diff --git a/src/lighteval/metrics/harness_compatibility/truthful_qa.py b/src/lighteval/metrics/harness_compatibility/truthful_qa.py deleted file mode 100644 index d8cbc3662..000000000 --- a/src/lighteval/metrics/harness_compatibility/truthful_qa.py +++ /dev/null @@ -1,62 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import numpy as np - -from lighteval.metrics.metrics_sample import SampleLevelComputation -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.requests import Doc -from lighteval.utils.utils import as_list - - -# Comes from the harness -class TruthfulqaMCMetrics(SampleLevelComputation): - def compute(self, doc: Doc, model_response: ModelResponse): - gold_ixs = as_list(doc.gold_index) - choices_logprob = model_response.logprobs - - # The harness assumes that all items are gold before the last one, but that is not always the case - # For gold ix 5, 6, 8, the harness will look at the first "gap" (7) and consider that the following - # items are not gold (even though here, 8 is gold). Example at item 371 of the dataset. - # This is broken and will have to be fixed once we OSS this, by actually separating - # gold and not gold items for mc2 computations - len_mc1 = doc.specific["len_mc1"] - last_harness_gold = gold_ixs[1] - 1 # fake value to init the loop - for g in gold_ixs[1:]: # we ignore the first item, which is the gold for mc1 - if last_harness_gold == g - 1: - last_harness_gold = g - else: - break - # TODO: This completely ignores any normalization, but keeping it as is - mc2_last_gold_ix = last_harness_gold - len_mc1 + 1 - mc1_lls, mc2_lls = choices_logprob[:len_mc1], choices_logprob[len_mc1:] - return {"truthfulqa_mc1": self.mc1(mc1_lls), "truthfulqa_mc2": self.mc2(mc2_lls, mc2_last_gold_ix)} - - def mc1(self, lls): - # The gold answers in `mc1_targets` are always first (index = `0`). - return np.argmax(lls) == 0 - - def mc2(self, lls, split_idx): - ll_true, ll_false = lls[:split_idx], lls[split_idx:] - p_true, p_false = np.exp(np.array(ll_true)), np.exp(np.array(ll_false)) - p_true = p_true / (sum(p_true) + sum(p_false)) - return sum(p_true) diff --git a/src/lighteval/metrics/imports/__init__.py b/src/lighteval/metrics/imports/__init__.py deleted file mode 100644 index a732db8d0..000000000 --- a/src/lighteval/metrics/imports/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. diff --git a/src/lighteval/metrics/imports/bert_scorer.py b/src/lighteval/metrics/imports/bert_scorer.py deleted file mode 100644 index db9c16c34..000000000 --- a/src/lighteval/metrics/imports/bert_scorer.py +++ /dev/null @@ -1,486 +0,0 @@ -# MIT License - -# Copyright (c) 2019 Tianyi Zhang, Varsha Kishore, Felix Wu, Kilian Q. -# Weinberger, and Yoav Artzi and The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -"""Simplified version of the BertScorer lib - we only import what we need.""" - -import logging -import os -import time -from collections import defaultdict - -import pandas as pd -import torch -from torch.nn.utils.rnn import pad_sequence -from transformers import AutoModel, AutoTokenizer - - -logger = logging.getLogger(__name__) - - -def padding(arr, pad_token, dtype=torch.long): - lens = torch.LongTensor([len(a) for a in arr]) - max_len = lens.max().item() - padded = torch.ones(len(arr), max_len, dtype=dtype) * pad_token - mask = torch.zeros(len(arr), max_len, dtype=torch.long) - for i, a in enumerate(arr): - padded[i, : lens[i]] = torch.tensor(a, dtype=dtype) - mask[i, : lens[i]] = 1 - return padded, lens, mask - - -def sent_encode(tokenizer, sent): - """Encoding as sentence based on the tokenizer""" - sent = sent.strip() - if sent == "": - return tokenizer.build_inputs_with_special_tokens([]) - return tokenizer.encode( - sent, - add_special_tokens=True, - max_length=tokenizer.model_max_length, - truncation=True, - ) - - -def bert_encode(model, x, attention_mask, all_layers=False): - model.eval() - with torch.no_grad(): - out = model(x, attention_mask=attention_mask, output_hidden_states=all_layers) - if all_layers: - emb = torch.stack(out[-1], dim=2) - else: - emb = out[0] - return emb - - -def collate_idf(arr, tokenizer, idf_dict, device="cuda:0"): - """Helper function that pads a list of sentences to have the same length and - loads idf score for words in the sentences. - - Args: - arr (list of str): sentences to process. - tokenizer: a tokenizer that takes a string and returns tokens. - idf_dict (dict): mapping a word piece index to its inverse document frequency. - device (str): device to use, e.g. 'cpu' or 'cuda'. - """ - arr = [sent_encode(tokenizer, a) for a in arr] - - idf_weights = [[idf_dict[i] for i in a] for a in arr] - - pad_token = tokenizer.pad_token_id - - padded, lens, mask = padding(arr, pad_token, dtype=torch.long) - padded_idf, _, _ = padding(idf_weights, 0, dtype=torch.float) - - padded = padded.to(device=device) - mask = mask.to(device=device) - lens = lens.to(device=device) - return padded, padded_idf, lens, mask - - -def get_bert_embedding( - all_sens, - model, - tokenizer, - idf_dict, - batch_size=-1, - device="cuda:0", - all_layers=False, -): - """Compute BERT embedding in batches. - - Args: - all_sens (list of str): sentences to encode. - model: a BERT model. - tokenizer: a BERT tokenizer corresponds to `model`. - idf_dict (dict): mapping a word piece index to its inverse document frequency. - batch_size (int): batch size for processing, -1 for all sentences. - device (str): device to use, e.g. 'cpu' or 'cuda'. - all_layers (bool): whether to return all layers or just the last layer. - - Returns: - tuple: A tuple containing (total_embedding, mask, padded_idf) - """ - padded_sens, padded_idf, _, mask = collate_idf(all_sens, tokenizer, idf_dict, device=device) - - if batch_size == -1: - batch_size = len(all_sens) - - embeddings = [] - with torch.no_grad(): - for i in range(0, len(all_sens), batch_size): - batch_embedding = bert_encode( - model, - padded_sens[i : i + batch_size], - attention_mask=mask[i : i + batch_size], - all_layers=all_layers, - ) - embeddings.append(batch_embedding) - del batch_embedding - - total_embedding = torch.cat(embeddings, dim=0) - - return total_embedding, mask, padded_idf - - -def greedy_cos_idf( - ref_embedding, - ref_masks, - ref_idf, - hyp_embedding, - hyp_masks, - hyp_idf, - all_layers=False, -): - """Compute greedy matching based on cosine similarity. - - Args: - ref_embedding (torch.Tensor): embeddings of reference sentences, BxKxd, - B: batch size, K: longest length, d: bert dimension. - ref_masks (torch.LongTensor): BxKxK, BERT attention mask for reference sentences. - ref_idf (torch.Tensor): BxK, idf score of each word piece in the reference sentence. - hyp_embedding (torch.Tensor): embeddings of candidate sentences, BxKxd, - B: batch size, K: longest length, d: bert dimension. - hyp_masks (torch.LongTensor): BxKxK, BERT attention mask for candidate sentences. - hyp_idf (torch.Tensor): BxK, idf score of each word piece in the candidate sentence. - all_layers (bool): whether to use all layers or just the last layer. - """ - ref_embedding.div_(torch.norm(ref_embedding, dim=-1).unsqueeze(-1)) - hyp_embedding.div_(torch.norm(hyp_embedding, dim=-1).unsqueeze(-1)) - - if all_layers: - B, _, L, D = hyp_embedding.size() - hyp_embedding = ( - hyp_embedding.transpose(1, 2).transpose(0, 1).contiguous().view(L * B, hyp_embedding.size(1), D) - ) - ref_embedding = ( - ref_embedding.transpose(1, 2).transpose(0, 1).contiguous().view(L * B, ref_embedding.size(1), D) - ) - batch_size = ref_embedding.size(0) - sim = torch.bmm(hyp_embedding, ref_embedding.transpose(1, 2)) - masks = torch.bmm(hyp_masks.unsqueeze(2).float(), ref_masks.unsqueeze(1).float()) - if all_layers: - masks = masks.unsqueeze(0).expand(L, -1, -1, -1).contiguous().view_as(sim) - else: - masks = masks.expand(batch_size, -1, -1).contiguous().view_as(sim) - - masks = masks.float().to(sim.device) - sim = sim * masks - - word_precision = sim.max(dim=2)[0] - word_recall = sim.max(dim=1)[0] - - hyp_idf.div_(hyp_idf.sum(dim=1, keepdim=True)) - ref_idf.div_(ref_idf.sum(dim=1, keepdim=True)) - precision_scale = hyp_idf.to(word_precision.device) - recall_scale = ref_idf.to(word_recall.device) - if all_layers: - precision_scale = precision_scale.unsqueeze(0).expand(L, B, -1).contiguous().view_as(word_precision) - recall_scale = recall_scale.unsqueeze(0).expand(L, B, -1).contiguous().view_as(word_recall) - P = (word_precision * precision_scale).sum(dim=1) - R = (word_recall * recall_scale).sum(dim=1) - F = 2 * P * R / (P + R) - - hyp_zero_mask = hyp_masks.sum(dim=1).eq(2) - ref_zero_mask = ref_masks.sum(dim=1).eq(2) - - if all_layers: - P = P.view(L, B) - R = R.view(L, B) - F = F.view(L, B) - - if torch.any(hyp_zero_mask): - logger.warning( - "Warning: Empty candidate sentence detected; setting raw BERTscores to 0.", - ) - P = P.masked_fill(hyp_zero_mask, 0.0) - R = R.masked_fill(hyp_zero_mask, 0.0) - - if torch.any(ref_zero_mask): - logger.warning("Empty reference sentence detected; setting raw BERTScores to 0.") - P = P.masked_fill(ref_zero_mask, 0.0) - R = R.masked_fill(ref_zero_mask, 0.0) - - F = F.masked_fill(torch.isnan(F), 0.0) - - return P, R, F - - -def bert_cos_score_idf( - model, - refs, - hyps, - tokenizer, - idf_dict, - verbose=False, - batch_size=64, - device="cuda:0", - all_layers=False, -): - """Compute BERTScore. - - Args: - model: a BERT model in `pytorch_pretrained_bert`. - refs (list of str): reference sentences. - hyps (list of str): candidate sentences. - tokenizer: a BERT tokenizer corresponds to `model`. - idf_dict: a dictionary mapping a word piece index to its inverse document frequency. - verbose (bool): turn on intermediate status update. - batch_size (int): bert score processing batch size. - device (str): device to use, e.g. 'cpu' or 'cuda'. - all_layers (bool): whether to use all layers or just the last layer. - """ - preds = [] - - def dedup_and_sort(l_item): - return sorted(set(l_item), key=lambda x: len(x.split(" ")), reverse=True) - - sentences = dedup_and_sort(refs + hyps) - embs = [] - iter_range = range(0, len(sentences), batch_size) - stats_dict = {} - for batch_start in iter_range: - sen_batch = sentences[batch_start : batch_start + batch_size] - embs, masks, padded_idf = get_bert_embedding( - sen_batch, model, tokenizer, idf_dict, device=device, all_layers=all_layers - ) - embs = embs.cpu() - masks = masks.cpu() - padded_idf = padded_idf.cpu() - for i, sen in enumerate(sen_batch): - sequence_len = masks[i].sum().item() - emb = embs[i, :sequence_len] - idf = padded_idf[i, :sequence_len] - stats_dict[sen] = (emb, idf) - - def pad_batch_stats(sen_batch, stats_dict, device): - stats = [stats_dict[s] for s in sen_batch] - emb, idf = zip(*stats) - emb = [e.to(device) for e in emb] - idf = [i.to(device) for i in idf] - lens = [e.size(0) for e in emb] - emb_pad = pad_sequence(emb, batch_first=True, padding_value=2.0) - idf_pad = pad_sequence(idf, batch_first=True) - - def length_to_mask(lens): - lens = torch.tensor(lens, dtype=torch.long) - max_len = max(lens) - base = torch.arange(max_len, dtype=torch.long).expand(len(lens), max_len) - return base < lens.unsqueeze(1) - - pad_mask = length_to_mask(lens).to(device) - return emb_pad, pad_mask, idf_pad - - device = next(model.parameters()).device - iter_range = range(0, len(refs), batch_size) - - with torch.no_grad(): - for batch_start in iter_range: - batch_refs = refs[batch_start : batch_start + batch_size] - batch_hyps = hyps[batch_start : batch_start + batch_size] - ref_stats = pad_batch_stats(batch_refs, stats_dict, device) - hyp_stats = pad_batch_stats(batch_hyps, stats_dict, device) - - P, R, F1 = greedy_cos_idf(*ref_stats, *hyp_stats, all_layers) - preds.append(torch.stack((P, R, F1), dim=-1).cpu()) - preds = torch.cat(preds, dim=1 if all_layers else 0) - return preds - - -class BERTScorer: - """BERTScore Scorer Object.""" - - def __init__( - self, - model_type=None, - num_layers=None, - batch_size=64, - nthreads=4, - all_layers=False, - idf=False, - device=None, - lang=None, - rescale_with_baseline=False, - baseline_path=None, - ): - """Initialize BERTScorer. - - Args: - model_type (str): Contextual embedding model specification, default using the suggested - model for the target language; has to specify at least one of - `model_type` or `lang`. - num_layers (int): The layer of representation to use. - Default using the number of layer tuned on WMT16 correlation data. - idf (bool): A boolean to specify whether to use idf or not (this should be True even if `idf_sents` is given). - device (str): On which the contextual embedding model will be allocated on. - If this argument is None, the model lives on cuda:0 if cuda is available. - batch_size (int): BERT score processing batch size. - nthreads (int): Number of threads. - all_layers (bool): Whether to use all layers or just the last layer. - lang (str): Language of the sentences; has to specify - at least one of `model_type` or `lang`. `lang` needs to be - specified when `rescale_with_baseline` is True. - rescale_with_baseline (bool): Rescale bertscore with pre-computed baseline. - baseline_path (str): Customized baseline file. - """ - assert lang is not None or model_type is not None, "Either lang or model_type should be specified" - - if rescale_with_baseline: - assert lang is not None, "Need to specify Language when rescaling with baseline" - - if device is None: - self.device = "cuda" if torch.cuda.is_available() else "cpu" - else: - self.device = device - - self._lang = lang - self._rescale_with_baseline = rescale_with_baseline - self._idf = idf - self.batch_size = batch_size - self.nthreads = nthreads - self.all_layers = all_layers - - self._model_type = model_type - self._num_layers = num_layers - - # Model and tokenizer are lazily loaded in `score()`. - self._tokenizer = None - self._model = None - - self._idf_dict = None - - self._baseline_vals = None - self.baseline_path = baseline_path - if self.baseline_path is None: - self.baseline_path = os.path.join( - os.path.dirname(__file__), - f"rescale_baseline/{self.lang}/{self.model_type}.tsv", - ) - - @property - def lang(self): - return self._lang - - @property - def idf(self): - return self._idf - - @property - def model_type(self): - return self._model_type - - @property - def num_layers(self): - return self._num_layers - - @property - def rescale_with_baseline(self): - return self._rescale_with_baseline - - @property - def baseline_vals(self): - if self._baseline_vals is None: - if os.path.isfile(self.baseline_path): - if not self.all_layers: - self._baseline_vals = torch.from_numpy( - pd.read_csv(self.baseline_path).iloc[self.num_layers].to_numpy() - )[1:].float() - else: - self._baseline_vals = ( - torch.from_numpy(pd.read_csv(self.baseline_path).to_numpy())[:, 1:].unsqueeze(1).float() - ) - else: - raise ValueError(f"Baseline not Found for {self.model_type} on {self.lang} at {self.baseline_path}") - - return self._baseline_vals - - def score(self, cands, refs, verbose=False, batch_size=64, return_hash=False): - """Args: - - :param: `cands` (list of str): candidate sentences - - :param: `refs` (list of str or list of list of str): reference sentences - - Return: - - :param: `(P, R, F)`: each is of shape (N); N = number of input - candidate reference pairs. if returning hashcode, the - output will be ((P, R, F), hashcode). If a candidate have - multiple references, the returned score of this candidate is - the *best* score among all references. - """ - if self._model is None: - logger.info(f"Loading BERTScorer model `{self._model_type}`") - self._tokenizer = AutoTokenizer.from_pretrained(self._model_type) - self._model = AutoModel.from_pretrained(self._model_type) - self._model.eval() - self._model.to(self.device) - - ref_group_boundaries = None - if not isinstance(refs[0], str): - ref_group_boundaries = [] - ori_cands, ori_refs = cands, refs - cands, refs = [], [] - count = 0 - for cand, ref_group in zip(ori_cands, ori_refs): - cands += [cand] * len(ref_group) - refs += ref_group - ref_group_boundaries.append((count, count + len(ref_group))) - count += len(ref_group) - - if verbose: - logger.info("calculating scores...") - start = time.perf_counter() - - if self.idf: - assert self._idf_dict, "IDF weights are not computed" - idf_dict = self._idf_dict - else: - idf_dict = defaultdict(lambda: 1.0) - idf_dict[self._tokenizer.sep_token_id] = 0 - idf_dict[self._tokenizer.cls_token_id] = 0 - - all_preds = bert_cos_score_idf( - self._model, - refs, - cands, - self._tokenizer, - idf_dict, - verbose=verbose, - device=self.device, - batch_size=batch_size, - all_layers=self.all_layers, - ).cpu() - - if ref_group_boundaries is not None: - max_preds = [] - for start, end in ref_group_boundaries: - max_preds.append(all_preds[start:end].max(dim=0)[0]) - all_preds = torch.stack(max_preds, dim=0) - - if self.rescale_with_baseline: - all_preds = (all_preds - self.baseline_vals) / (1 - self.baseline_vals) - - out = all_preds[..., 0], all_preds[..., 1], all_preds[..., 2] # P, R, F - - if verbose: - time_diff = time.perf_counter() - start - logger.info(f"done in {time_diff:.2f} seconds, {len(refs) / time_diff:.2f} sentences/sec") - - return out diff --git a/src/lighteval/metrics/imports/data_stats_metric.py b/src/lighteval/metrics/imports/data_stats_metric.py deleted file mode 100644 index a007bcf5d..000000000 --- a/src/lighteval/metrics/imports/data_stats_metric.py +++ /dev/null @@ -1,156 +0,0 @@ -# MIT License - -# Copyright (c) 2021 Alex Fabbri, Wojciech Kryściński, Bryan -# McCann, Caiming Xiong, Richard Socher, and Dragomir Radev and The HuggingFace -# Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -# pylint: disable=C0103,W0221,W0106 -# Replace summ_eval.data_stats_metric -import logging -from collections import Counter -from multiprocessing import Pool -from typing import Literal - -from lighteval.metrics.imports.data_stats_utils import Fragments -from lighteval.utils.imports import Extra, requires - - -logger = logging.getLogger(__name__) - -LANGUAGE_TO_SPACY_MODEL_MAP = { - "en": "en_core_web_sm", - "de": "de_core_news_sm", - "fr": "fr_core_news_sm", - "it": "it_core_news_sm", -} - - -class Metric: - def evaluate_example(self, summary, reference): - raise NotImplementedError - - def evaluate_batch(self, summaries, references, aggregate=True): - raise NotImplementedError - - -def find_ngrams(input_list, n): - return zip(*[input_list[i:] for i in range(n)]) - - -@requires(Extra.MULTILINGUAL) -class DataStatsMetric(Metric): - def __init__( - self, - n_gram: int = 3, - n_workers: int = 24, - case: bool = False, - tokenize: bool = True, - language: Literal["en", "de", "fr", "it"] = "en", - ): - """ - Data Statistics metric - Makes use of Newsroom code: \ - https://github.com/lil-lab/newsroom/blob/master/newsroom/analyze/fragments.py - Calculates extractive statistics such as coverage, density, compression as - defined in Newsroom paper as well as the percentage of novel n-grams in the - summary vs the input text and the percentage of n-grams in the summary which are - repeated - - NOTE: these statistics are meant to be calculated with respect to the source text - (e.g. news article) as opposed to the reference. - - Args: - n_gram (int): compute statistics for n-grams up to and including this length. - n_workers (int): number of processes to use if using multiprocessing. - case (bool): whether to lowercase input before calculating statistics. - tokenize (bool): whether to tokenize the input; otherwise assumes that the input - is a string of space-separated tokens. - language (Literal["en", "de", "fr", "it"]): the language of the input text. This - determines the spaCy model used for tokenization. Currently supports English, - German, French, and Italian. - """ - import spacy - - self.n_gram = n_gram - self.n_workers = n_workers - self.case = case - self.tokenize = tokenize - self.language = language - self.nlp = None - - spacy_model = LANGUAGE_TO_SPACY_MODEL_MAP.get(self.language, "en_core_web_sm") - try: - self.nlp = spacy.load(spacy_model) - except OSError: - logger.info(f"Downloading the spacy {spacy_model} model\n(don't worry, this will only happen once)") - from spacy.cli import download - - download(spacy_model) - self.nlp = spacy.load(spacy_model) - - def evaluate_example(self, summary, input_text): - if self.tokenize: - input_text = self.nlp(input_text, disable=["tagger", "parser", "ner", "textcat"]) - input_text = [tok.text for tok in input_text] - summary = self.nlp(summary, disable=["tagger", "parser", "ner", "textcat"]) - summary = [tok.text for tok in summary] - fragments = Fragments(summary, input_text, case=self.case) - coverage = fragments.coverage() - density = fragments.density() - compression = fragments.compression() - score_dict = {"coverage": coverage, "density": density, "compression": compression} - tokenized_summary = fragments.summary - tokenized_text = fragments.text - score_dict["summary_length"] = len(tokenized_summary) - for i in range(1, self.n_gram + 1): - input_ngrams = list(find_ngrams(tokenized_text, i)) - summ_ngrams = list(find_ngrams(tokenized_summary, i)) - input_ngrams_set = set(input_ngrams) - summ_ngrams_set = set(summ_ngrams) - intersect = summ_ngrams_set.intersection(input_ngrams_set) - try: - score_dict[f"percentage_novel_{i}-gram"] = (len(summ_ngrams_set) - len(intersect)) / float( - len(summ_ngrams_set) - ) - ngramCounter = Counter() - ngramCounter.update(summ_ngrams) - repeated = [key for key, val in ngramCounter.items() if val > 1] - score_dict[f"percentage_repeated_{i}-gram_in_summ"] = len(repeated) / float(len(summ_ngrams_set)) - except ZeroDivisionError: - continue - return score_dict - - def evaluate_batch(self, summaries, input_texts, aggregate=True): - corpus_score_dict = Counter() - p = Pool(processes=self.n_workers) - results = p.starmap(self.evaluate_example, zip(summaries, input_texts)) - p.close() - if aggregate: - [corpus_score_dict.update(x) for x in results] - for key in corpus_score_dict.keys(): - corpus_score_dict[key] /= float(len(input_texts)) - return corpus_score_dict - else: - return results - - @property - def supports_multi_ref(self): - return False diff --git a/src/lighteval/metrics/imports/data_stats_utils.py b/src/lighteval/metrics/imports/data_stats_utils.py deleted file mode 100644 index c5ae00725..000000000 --- a/src/lighteval/metrics/imports/data_stats_utils.py +++ /dev/null @@ -1,190 +0,0 @@ -# pylint: disable=C0103 # noqa: CPY001 -################################################################################ -# Taken from https://github.com/lil-lab/newsroom/blob/master/newsroom/analyze/fragments.py - -from collections import namedtuple as _namedtuple - - -def normalize(tokens, case=False): - """Lowercases and turns tokens into distinct words. - - Returns: - list[str]: List of normalized tokens - """ - return [str(t).lower() if not case else str(t) for t in tokens] - - -################################################################################ - - -class Fragments: - Match = _namedtuple("Match", ("summary", "text", "length")) - - def __init__(self, summary, text, case=False): - # self._tokens = tokenize - - if isinstance(summary, str): - self.summary = summary.split() - else: - self.summary = summary - if isinstance(text, str): - self.text = text.split() - else: - self.text = text - - self._norm_summary = normalize(self.summary, case) - self._norm_text = normalize(self.text, case) - - self._match(self._norm_summary, self._norm_text) - - def overlaps(self): - """Return a list of Fragments.Match objects between summary and text. - This is a list of named tuples of the form (summary, text, length): - - - summary (int): the start index of the match in the summary - - text (int): the start index of the match in the reference - - length (int): the length of the extractive fragment - - """ - return self._matches - - def strings(self, min_length=0, summary_base=True): - """Return a list of explicit match strings between the summary and reference. - Note that this will be in the same format as the strings are input. This is - important to remember if tokenization is done manually. If tokenization is - specified automatically on the raw strings, raw strings will automatically - be returned rather than SpaCy tokenized sequences. - - Args: - min_length (int): filter out overlaps shorter than this (default = 0). - summary_base (bool): strings are based of summary text (default = True). - - Returns: - list: list of overlaps, where overlaps are strings or token sequences. - """ - # Compute the strings against the summary or the text? - - base = self.summary if summary_base else self.text - - # Generate strings, filtering out strings below the minimum length. - - strings = [base[i : i + length] for i, j, length in self.overlaps() if length > min_length] - - # By default, we just return the tokenization being used. - # But if they user wants a raw string, then we convert. - # Mostly, this will be used along with spacy. - - # if self._tokens and raw: - - # for i, s in enumerate(strings): - # strings[i] = str(s) - - # Return the list of strings. - - return strings - - def coverage(self, summary_base=True): - """Return the COVERAGE score of the summary and text. - - Args: - summary_base (bool): use summary as numerator (default = True). - - Returns: - float: decimal COVERAGE score within [0, 1]. - """ - numerator = sum(o.length for o in self.overlaps()) - - if summary_base: - denominator = len(self.summary) - else: - denominator = len(self.text) - - if denominator == 0: - return 0 - else: - return numerator / denominator - - def density(self, summary_base=True): - """Return the DENSITY score of summary and text. - - Args: - summary_base (bool): use summary as numerator (default = True). - - Returns: - float: decimal DENSITY score within [0, ...]. - """ - numerator = sum(o.length**2 for o in self.overlaps()) - - if summary_base: - denominator = len(self.summary) - else: - denominator = len(self.text) - - if denominator == 0: - return 0 - else: - return numerator / denominator - - def compression(self, text_to_summary=True): - """Return compression ratio between summary and text. - - Args: - text_to_summary (bool): compute text/summary ratio (default = True). - - Returns: - float: decimal compression score within [0, ...]. - """ - ratio = [len(self.text), len(self.summary)] - - try: - if text_to_summary: - return ratio[0] / ratio[1] - else: - return ratio[1] / ratio[0] - - except ZeroDivisionError: - return 0 - - def _match(self, a, b): - """Raw procedure for matching summary in text, described in paper.""" - self._matches = [] - - a_start = b_start = 0 - - while a_start < len(a): - best_match = None - best_match_length = 0 - - while b_start < len(b): - if a[a_start] == b[b_start]: - a_end = a_start - b_end = b_start - - while a_end < len(a) and b_end < len(b) and b[b_end] == a[a_end]: - b_end += 1 - a_end += 1 - - length = a_end - a_start - - if length > best_match_length: - best_match = Fragments.Match(a_start, b_start, length) - best_match_length = length - - b_start = b_end - - else: - b_start += 1 - - b_start = 0 - - if best_match: - if best_match_length > 0: - self._matches.append(best_match) - - a_start += best_match_length - - else: - a_start += 1 - - -################################################################################ diff --git a/src/lighteval/metrics/imports/summac.py b/src/lighteval/metrics/imports/summac.py deleted file mode 100644 index bda317b79..000000000 --- a/src/lighteval/metrics/imports/summac.py +++ /dev/null @@ -1,319 +0,0 @@ -# noqa: CPY001 -# Source: https://github.com/tingofurro/summac -# Apache 2 license -############################################### - -import json -import logging -import os -import time - -import nltk -import numpy as np -import torch -import tqdm -from transformers import AutoModelForSequenceClassification, AutoTokenizer - - -logger = logging.getLogger(__name__) - -# GPU-related business - - -def get_freer_gpu(): - os.system("nvidia-smi -q -d Memory |grep -A4 GPU|grep Free >tmp_smi") - memory_available = [int(x.split()[2]) + 5 * i for i, x in enumerate(open("tmp_smi", "r").readlines())] - os.remove("tmp_smi") - return np.argmax(memory_available) - - -def any_gpu_with_space(gb_needed): - os.system("nvidia-smi -q -d Memory |grep -A4 GPU|grep Free >tmp_smi") - memory_available = [float(x.split()[2]) / 1024.0 for i, x in enumerate(open("tmp_smi", "r").readlines())] - os.remove("tmp_smi") - return any(mem >= gb_needed for mem in memory_available) - - -def wait_free_gpu(gb_needed): - while not any_gpu_with_space(gb_needed): - time.sleep(30) - - -def select_freer_gpu(): - freer_gpu = str(get_freer_gpu()) - logger.info("Will use GPU: %s" % (freer_gpu)) - os.environ["CUDA_LAUNCH_BLOCKING"] = "1" - os.environ["CUDA_VISIBLE_DEVICES"] = "" + freer_gpu - return freer_gpu - - -def batcher(iterator, batch_size=4, progress=False): - if progress: - iterator = tqdm.tqdm(iterator) - - batch = [] - for elem in iterator: - batch.append(elem) - if len(batch) == batch_size: - final_batch = batch - batch = [] - yield final_batch - if len(batch) > 0: # Leftovers - yield batch - - -model_map = { - "snli-base": {"model_card": "boychaboy/SNLI_roberta-base", "entailment_idx": 0, "contradiction_idx": 2}, - "snli-large": {"model_card": "boychaboy/SNLI_roberta-large", "entailment_idx": 0, "contradiction_idx": 2}, - "mnli-base": {"model_card": "microsoft/deberta-base-mnli", "entailment_idx": 2, "contradiction_idx": 0}, - "mnli": {"model_card": "roberta-large-mnli", "entailment_idx": 2, "contradiction_idx": 0}, - "anli": { - "model_card": "ynie/roberta-large-snli_mnli_fever_anli_R1_R2_R3-nli", - "entailment_idx": 0, - "contradiction_idx": 2, - }, - "vitc-base": {"model_card": "tals/albert-base-vitaminc-mnli", "entailment_idx": 0, "contradiction_idx": 1}, - "vitc": {"model_card": "tals/albert-xlarge-vitaminc-mnli", "entailment_idx": 0, "contradiction_idx": 1}, - "vitc-only": {"model_card": "tals/albert-xlarge-vitaminc", "entailment_idx": 0, "contradiction_idx": 1}, - # "decomp": 0, -} - - -def card_to_name(card): - card2name = {v["model_card"]: k for k, v in model_map.items()} - if card in card2name: - return card2name[card] - return card - - -def name_to_card(name): - if name in model_map: - return model_map[name]["model_card"] - return name - - -def get_neutral_idx(ent_idx, con_idx): - return list({0, 1, 2} - {ent_idx, con_idx})[0] - - -class SummaCImager: - def __init__( - self, model_name="mnli", granularity="paragraph", use_cache=True, max_doc_sents=100, device="cuda", **kwargs - ): - self.grans = granularity.split("-") - - assert ( - all(gran in ["paragraph", "sentence", "document", "2sents", "mixed"] for gran in self.grans) - and len(self.grans) <= 2 - ), "Unrecognized `granularity` %s" % (granularity) - assert model_name in model_map.keys(), "Unrecognized model name: `%s`" % (model_name) - - self.model_name = model_name - if model_name != "decomp": - self.model_card = name_to_card(model_name) - self.entailment_idx = model_map[model_name]["entailment_idx"] - self.contradiction_idx = model_map[model_name]["contradiction_idx"] - self.neutral_idx = get_neutral_idx(self.entailment_idx, self.contradiction_idx) - - self.granularity = granularity - self.use_cache = use_cache - self.cache_folder = "/export/share/plaban/summac_cache/" - - self.max_doc_sents = max_doc_sents - self.max_input_length = 500 - self.device = device - self.cache = {} - self.model = None # Lazy loader - - def load_nli(self): - if self.model_name == "decomp": - from allennlp.predictors.predictor import Predictor - - self.model = Predictor.from_path( - "https://storage.googleapis.com/allennlp-public-models/decomposable-attention-elmo-2020.04.09.tar.gz", - cuda_device=0, - ) - - else: - self.tokenizer = AutoTokenizer.from_pretrained(self.model_card) - self.model = AutoModelForSequenceClassification.from_pretrained(self.model_card).eval() - self.model.to(self.device).half() - - def split_sentences(self, text): - sentences = nltk.tokenize.sent_tokenize(text) - sentences = [sent for sent in sentences if len(sent) > 10] - return sentences - - def split_2sents(self, text): - sentences = nltk.tokenize.sent_tokenize(text) - sentences = [sent for sent in sentences if len(sent) > 10] - two_sents = [" ".join(sentences[i : (i + 2)]) for i in range(len(sentences))] - return two_sents - - def split_paragraphs(self, text): - if text.count("\n\n") > 0: - paragraphs = [p.strip() for p in text.split("\n\n")] - else: - paragraphs = [p.strip() for p in text.split("\n")] - return [p for p in paragraphs if len(p) > 10] - - def split_text(self, text, granularity="sentence"): - if granularity == "document": - return [text] - elif granularity == "paragraph": - return self.split_paragraphs(text) - elif granularity == "sentence": - return self.split_sentences(text) - elif granularity == "2sents": - return self.split_2sents(text) - elif granularity == "mixed": - return self.split_sentences(text) + self.split_paragraphs(text) - - def build_image(self, original, generated): - cache_key = (original, generated) - if self.use_cache and cache_key in self.cache: - cached_image = self.cache[cache_key] - cached_image = cached_image[:, : self.max_doc_sents, :] - return cached_image - - if len(self.grans) == 1: - gran_doc, gran_sum = self.grans[0], self.grans[0] - else: - gran_doc, gran_sum = self.grans[0], self.grans[1] - - original_chunks = self.split_text(original, granularity=gran_doc)[: self.max_doc_sents] - generated_chunks = self.split_text(generated, granularity=gran_sum) - - N_ori = len(original_chunks) - N_gen = len(generated_chunks) - - if N_ori == 0 or N_gen == 0: - return np.zeros((3, 1, 1)) - # assert (N_ori > 0 and N_gen > 0), "One of the inputs has no chunks" - - image = np.zeros((3, N_ori, N_gen)) - - if self.model is None: - self.load_nli() - - dataset = [ - {"premise": original_chunks[i], "hypothesis": generated_chunks[j], "doc_i": i, "gen_i": j} - for i in range(N_ori) - for j in range(N_gen) - ] - for batch in batcher(dataset, batch_size=20): - if self.model_name == "decomp": - batch_evids, batch_conts, batch_neuts = [], [], [] - batch_json = [{"premise": d["premise"], "hypothesis": d["hypothesis"]} for d in batch] - model_outs = self.model.predict_batch_json(batch_json) - for out in model_outs: - probs = out["label_probs"] - batch_evids.append(probs[0]) - batch_conts.append(probs[1]) - batch_neuts.append(probs[2]) - - else: - batch_prems = [b["premise"] for b in batch] - batch_hypos = [b["hypothesis"] for b in batch] - batch_tokens = self.tokenizer.batch_encode_plus( - list(zip(batch_prems, batch_hypos)), - padding=True, - truncation=True, - max_length=self.max_input_length, - return_tensors="pt", - ) - batch_tokens = {k: v.to(self.device) for k, v in batch_tokens.items()} - with torch.no_grad(): - model_outputs = self.model(**batch_tokens) - - batch_probs = torch.nn.functional.softmax(model_outputs["logits"], dim=-1) - batch_evids = batch_probs[:, self.entailment_idx].tolist() - batch_conts = batch_probs[:, self.contradiction_idx].tolist() - batch_neuts = batch_probs[:, self.neutral_idx].tolist() - - for b, evid, cont, neut in zip(batch, batch_evids, batch_conts, batch_neuts): - image[0, b["doc_i"], b["gen_i"]] = evid - image[1, b["doc_i"], b["gen_i"]] = cont - image[2, b["doc_i"], b["gen_i"]] = neut - - if self.use_cache: - self.cache[cache_key] = image - return image - - def get_cache_file(self): - return os.path.join(self.cache_folder, "cache_%s_%s.json" % (self.model_name, self.granularity)) - - def save_cache(self): - cache_cp = {"[///]".join(k): v.tolist() for k, v in self.cache.items()} - with open(self.get_cache_file(), "w") as f: - json.dump(cache_cp, f) - - def load_cache(self): - cache_file = self.get_cache_file() - if os.path.isfile(cache_file): - with open(cache_file, "r") as f: - cache_cp = json.load(f) - self.cache = {tuple(k.split("[///]")): np.array(v) for k, v in cache_cp.items()} - - -class SummaCZS: - def __init__( - self, - model_name="mnli", - granularity="paragraph", - op1="max", - op2="mean", - use_ent=True, - use_con=True, - imager_load_cache=True, - device="cuda", - **kwargs, - ): - assert op2 in ["min", "mean", "max"], "Unrecognized `op2`" - assert op1 in ["max", "mean", "min"], "Unrecognized `op1`" - - self.imager = SummaCImager(model_name=model_name, granularity=granularity, device=device, **kwargs) - if imager_load_cache: - self.imager.load_cache() - self.op2 = op2 - self.op1 = op1 - self.use_ent = use_ent - self.use_con = use_con - - def save_imager_cache(self): - self.imager.save_cache() - - def score_one(self, original, generated): - image = self.imager.build_image(original, generated) - - ent_scores = np.max(image[0], axis=0) - co_scores = np.max(image[1], axis=0) - if self.op1 == "mean": - ent_scores = np.mean(image[0], axis=0) - co_scores = np.mean(image[1], axis=0) - elif self.op1 == "min": - ent_scores = np.min(image[0], axis=0) - co_scores = np.min(image[1], axis=0) - - if self.use_ent and self.use_con: - scores = ent_scores - co_scores - elif self.use_ent: - scores = ent_scores - elif self.use_con: - scores = 1.0 - co_scores - - final_score = np.mean(scores) - if self.op2 == "min": - final_score = np.min(scores) - elif self.op2 == "max": - final_score = np.max(scores) - - return {"score": final_score, "image": image} - - def score(self, sources, generateds, **kwargs): - output = {"scores": [], "images": []} - for source, gen in zip(sources, generateds): - score = self.score_one(source, gen) - output["scores"].append(score["score"]) - output["images"].append(score["image"]) - return output diff --git a/src/lighteval/metrics/metrics.py b/src/lighteval/metrics/metrics.py deleted file mode 100644 index 82cfbb706..000000000 --- a/src/lighteval/metrics/metrics.py +++ /dev/null @@ -1,633 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -from copy import deepcopy - -import numpy as np -from aenum import Enum -from inspect_ai.scorer import Score, Target, accuracy, scorer, stderr -from inspect_ai.solver import TaskState - -from lighteval.metrics.dynamic_metrics import MultilingualExtractiveMatchMetric -from lighteval.metrics.harness_compatibility.drop import DropMetrics -from lighteval.metrics.harness_compatibility.truthful_qa import TruthfulqaMCMetrics -from lighteval.metrics.metrics_corpus import ( - CorpusLevelF1Score, - CorpusLevelPerplexityMetric, - CorpusLevelTranslationMetric, - MatthewsCorrCoef, -) -from lighteval.metrics.metrics_sample import ( - BLEU, - BLEURT, - MRR, - ROUGE, - AccGoldLikelihood, - AvgAtN, - BertScore, - ExactMatches, - Extractiveness, - F1_score, - Faithfulness, - GPassAtK, - JudgeLLMSimpleQA, - LoglikelihoodAcc, - MajAtN, - PassAtK, - Recall, - StringDistance, -) -from lighteval.metrics.normalizations import bigbench_normalizer, remove_braces, remove_braces_and_strip -from lighteval.metrics.sample_preparator import ( - GenerativePreparator, - LoglikelihoodPreparator, - PerplexityPreparator, - TargetPerplexityPreparator, -) -from lighteval.metrics.utils.extractive_match_utils import ( - ExprExtractionConfig, - IndicesExtractionConfig, - LatexExtractionConfig, - extract_target_from_pred, - get_extraction_regexes_inspect, -) -from lighteval.metrics.utils.metric_utils import ( - CorpusLevelMetric, - CorpusLevelMetricGrouping, - SampleLevelMetric, - SampleLevelMetricGrouping, - SamplingMethod, -) -from lighteval.utils.language import Language - - -@scorer(metrics=[accuracy()]) -def math_scorer(): - gold_extraction_target = (ExprExtractionConfig(),) - pred_extraction_target = (ExprExtractionConfig(), LatexExtractionConfig(boxed_match_priority=0)) - language = Language.ENGLISH - fallback_mode = "first_match" - extraction_mode = "first_match" - timeout_seconds = 5 - - gold_extraction_regexes = get_extraction_regexes_inspect(gold_extraction_target, language, len_choices=1) - pred_extraction_regexes = get_extraction_regexes_inspect(pred_extraction_target, language, len_choices=1) - - async def score(state: TaskState, target: Target): - extracted_predictions = extract_target_from_pred( - state.output.completion, pred_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds - ) - extracted_gold = extract_target_from_pred( - target.text, gold_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds - ) - return Score( - # Correct or Incorrect, used by inspect-ai backend - value="C" if extracted_predictions == extracted_gold else "I", - explanation=state.output.completion, - answer=str(extracted_predictions), - ) - - return score - - -@scorer(metrics=[accuracy(), stderr()]) -def multichoice_scorer(): - language = Language.ENGLISH - gold_extraction_target = ( - IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True), - ) - pred_extraction_target = ( - IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True), - ) - fallback_mode = "first_match" - extraction_mode = "first_match" - timeout_seconds = 5 - - gold_extraction_regexes = get_extraction_regexes_inspect(gold_extraction_target, language) - pred_extraction_regexes = get_extraction_regexes_inspect(pred_extraction_target, language) - - async def score(state: TaskState, target: Target): - extracted_predictions = extract_target_from_pred( - state.output.completion, pred_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds - ) - extracted_gold = extract_target_from_pred( - target.text, gold_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds - ) - return Score( - # Correct or Incorrect, used by inspect-ai backend - value="C" if extracted_predictions == extracted_gold else "I", - explanation=state.output.completion, - answer=str(extracted_predictions), - ) - - return score - - -class Metrics(Enum): - acc_golds_likelihood = SampleLevelMetric( # todo: we need a better name for this! - metric_name="acc", - sample_level_fn=AccGoldLikelihood(), - category=SamplingMethod.LOGPROBS, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - avg_at_n = SampleLevelMetric( - metric_name="avg@n", - sample_level_fn=AvgAtN(strip_strings=True), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - avg_at_n_math = SampleLevelMetric( - metric_name="avg@n", - sample_level_fn=AvgAtN( - sample_scoring_function=MultilingualExtractiveMatchMetric( - language=Language.ENGLISH, - gold_extraction_target=[ExprExtractionConfig(), LatexExtractionConfig()], - pred_extraction_target=[ExprExtractionConfig(), LatexExtractionConfig()], - precision=6, - ), - ), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - bert_score = SampleLevelMetricGrouping( - metric_name=["BERTScore-P", "BERTScore-R", "BERTScore-F"], - sample_level_fn=BertScore(normalize_gold=remove_braces, normalize_pred=remove_braces_and_strip), - category=SamplingMethod.GENERATIVE, - corpus_level_fn={"BERTScore-P": np.mean, "BERTScore-R": np.mean, "BERTScore-F": np.mean}, - higher_is_better={"BERTScore-P": True, "BERTScore-R": True, "BERTScore-F": True}, - ) - bits_per_byte = CorpusLevelMetric( - metric_name="bits_per_byte", - sample_level_fn=PerplexityPreparator(units_type="bytes"), - category=SamplingMethod.PERPLEXITY, - corpus_level_fn=CorpusLevelPerplexityMetric("bits_per_byte"), - higher_is_better=False, - ) - bleu = CorpusLevelMetric( - metric_name="bleu", - sample_level_fn=GenerativePreparator(), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=CorpusLevelTranslationMetric("bleu"), - higher_is_better=True, - ) - bleu_1 = SampleLevelMetric( - metric_name="bleu_1", - sample_level_fn=BLEU(n_gram=1), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - bleu_4 = SampleLevelMetric( - metric_name="bleu_4", - sample_level_fn=BLEU(n_gram=4), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - - bleurt = SampleLevelMetric( - metric_name="bleurt", - sample_level_fn=BLEURT(), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - byte_perplexity = CorpusLevelMetric( - metric_name="byte_perplexity", - sample_level_fn=PerplexityPreparator(units_type="bytes"), - category=SamplingMethod.PERPLEXITY, - corpus_level_fn=CorpusLevelPerplexityMetric("weighted_perplexity"), - higher_is_better=False, - ) - chrf = CorpusLevelMetric( - metric_name="chrf", - sample_level_fn=GenerativePreparator(), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=CorpusLevelTranslationMetric("chrf"), - higher_is_better=True, - ) - chrf_plus = CorpusLevelMetric( - metric_name="chrf++", - sample_level_fn=GenerativePreparator(), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=CorpusLevelTranslationMetric("chrf++"), - higher_is_better=True, - ) - copyright = SampleLevelMetricGrouping( - metric_name=["longest_common_prefix_length", "edit_distance", "edit_similarity"], - sample_level_fn=StringDistance( - metric_types=["longest_common_prefix_length", "edit_distance", "edit_similarity"], strip_prediction=True - ), - category=SamplingMethod.GENERATIVE, - corpus_level_fn={"longest_common_prefix_length": max, "edit_distance": min, "edit_similarity": max}, - higher_is_better={"longest_common_prefix_length": True, "edit_distance": False, "edit_similarity": True}, - ) - drop = SampleLevelMetricGrouping( - metric_name=["em", "f1"], - sample_level_fn=DropMetrics(), - category=SamplingMethod.GENERATIVE, - corpus_level_fn={"em": max, "f1": max}, - higher_is_better={"em": True, "f1": True}, - ) - exact_match = SampleLevelMetric( - metric_name="em", - sample_level_fn=ExactMatches(strip_strings=True), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - expr_gold_metric = SampleLevelMetric( - metric_name="extractive_match", - sample_level_fn=MultilingualExtractiveMatchMetric( - language=Language.ENGLISH, - fallback_mode="first_match", - precision=5, - gold_extraction_target=(ExprExtractionConfig(),), - # Match boxed first before trying other regexes - pred_extraction_target=(ExprExtractionConfig(), LatexExtractionConfig(boxed_match_priority=0)), - aggregation_function=max, - ), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - extractiveness = SampleLevelMetricGrouping( - metric_name=["summarization_coverage", "summarization_density", "summarization_compression"], - sample_level_fn=Extractiveness( - normalize_input=remove_braces, normalize_pred=remove_braces_and_strip, input_column="text" - ), - category=SamplingMethod.GENERATIVE, - corpus_level_fn={ - "summarization_coverage": np.mean, - "summarization_density": np.mean, - "summarization_compression": np.mean, - }, - higher_is_better={ - "summarization_coverage": True, - "summarization_density": True, - "summarization_compression": True, - }, - ) - extractiveness_de = SampleLevelMetricGrouping( - metric_name=["summarization_coverage", "summarization_density", "summarization_compression"], - sample_level_fn=Extractiveness( - normalize_input=remove_braces, normalize_pred=remove_braces_and_strip, input_column="text", language="de" - ), - category=SamplingMethod.GENERATIVE, - corpus_level_fn={ - "summarization_coverage": np.mean, - "summarization_density": np.mean, - "summarization_compression": np.mean, - }, - higher_is_better={ - "summarization_coverage": True, - "summarization_density": True, - "summarization_compression": True, - }, - ) - extractiveness_fr = SampleLevelMetricGrouping( - metric_name=["summarization_coverage", "summarization_density", "summarization_compression"], - sample_level_fn=Extractiveness( - normalize_input=remove_braces, normalize_pred=remove_braces_and_strip, input_column="text", language="fr" - ), - category=SamplingMethod.GENERATIVE, - corpus_level_fn={ - "summarization_coverage": np.mean, - "summarization_density": np.mean, - "summarization_compression": np.mean, - }, - higher_is_better={ - "summarization_coverage": True, - "summarization_density": True, - "summarization_compression": True, - }, - ) - extractiveness_it = SampleLevelMetricGrouping( - metric_name=["summarization_coverage", "summarization_density", "summarization_compression"], - sample_level_fn=Extractiveness( - normalize_input=remove_braces, normalize_pred=remove_braces_and_strip, input_column="text", language="it" - ), - category=SamplingMethod.GENERATIVE, - corpus_level_fn={ - "summarization_coverage": np.mean, - "summarization_density": np.mean, - "summarization_compression": np.mean, - }, - higher_is_better={ - "summarization_coverage": True, - "summarization_density": True, - "summarization_compression": True, - }, - ) - f1_score = SampleLevelMetric( - metric_name="f1", - sample_level_fn=F1_score(), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - f1_score_macro = CorpusLevelMetric( - metric_name="f1", - sample_level_fn=GenerativePreparator(), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=CorpusLevelF1Score(average="macro"), - higher_is_better=True, - ) - f1_score_micro = CorpusLevelMetric( - metric_name="f1", - sample_level_fn=GenerativePreparator(), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=CorpusLevelF1Score(average="micro"), - higher_is_better=True, - ) - faithfulness = SampleLevelMetric( - metric_name="summac", - sample_level_fn=Faithfulness( - normalize_input=remove_braces, normalize_pred=remove_braces_and_strip, input_column="text" - ), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - g_pass_at_k = SampleLevelMetricGrouping( - metric_name="g-pass@k", - sample_level_fn=GPassAtK(strip_strings=True), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - g_pass_at_k_math = SampleLevelMetricGrouping( - metric_name="math-g-pass@k", - sample_level_fn=GPassAtK( - name_prefix="math", - strip_strings=True, - sample_scoring_function=MultilingualExtractiveMatchMetric( - language=Language.ENGLISH, - fallback_mode="first_match", - precision=5, - gold_extraction_target=(ExprExtractionConfig(),), - # Match boxed first before trying other regexes - pred_extraction_target=(ExprExtractionConfig(), LatexExtractionConfig(boxed_match_priority=0)), - aggregation_function=max, - ), - ), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - g_pass_at_k_latex = SampleLevelMetricGrouping( - metric_name="latex-g-pass@k", - sample_level_fn=GPassAtK( - name_prefix="latex", - strip_strings=True, - sample_scoring_function=MultilingualExtractiveMatchMetric( - language=Language.ENGLISH, - fallback_mode="first_match", - precision=5, - gold_extraction_target=(LatexExtractionConfig(),), - # Match boxed first before trying other regexes - pred_extraction_target=(ExprExtractionConfig(), LatexExtractionConfig(boxed_match_priority=0)), - aggregation_function=max, - ), - ), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - loglikelihood_acc = SampleLevelMetric( - metric_name="acc", - sample_level_fn=LoglikelihoodAcc(logprob_normalization=None), - category=SamplingMethod.LOGPROBS, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - loglikelihood_f1 = CorpusLevelMetric( - metric_name="loglikelihood_f1", - sample_level_fn=LoglikelihoodPreparator(), - category=SamplingMethod.LOGPROBS, - corpus_level_fn=CorpusLevelF1Score(None), - higher_is_better=True, - ) - maj_at_n = SampleLevelMetric( - metric_name="maj@n", - sample_level_fn=MajAtN(), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - mcc = CorpusLevelMetric( - metric_name="mcc", - sample_level_fn=LoglikelihoodPreparator(), - category=SamplingMethod.LOGPROBS, - corpus_level_fn=MatthewsCorrCoef(), - higher_is_better=True, - ) - mrr = SampleLevelMetric( - metric_name="mrr", - sample_level_fn=MRR(), - category=SamplingMethod.LOGPROBS, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - multi_f1_numeric = CorpusLevelMetric( - metric_name="mf1", - sample_level_fn=LoglikelihoodPreparator(is_single_token=True), - category=SamplingMethod.LOGPROBS, - corpus_level_fn=CorpusLevelF1Score(average="micro", num_classes=3), - higher_is_better=True, - ) - pass_at_k = SampleLevelMetric( - metric_name="pass@k", - sample_level_fn=PassAtK(strip_strings=True), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - pass_at_k_math = SampleLevelMetric( - metric_name="pass@k", - sample_level_fn=PassAtK( - strip_strings=True, - # Extracting mathematical expressions and latex expressions - sample_scoring_function=MultilingualExtractiveMatchMetric( - language=Language.ENGLISH, - gold_extraction_target=[ExprExtractionConfig(), LatexExtractionConfig()], - pred_extraction_target=[ExprExtractionConfig(), LatexExtractionConfig()], - precision=6, - ), - ), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - pass_at_k_letters = SampleLevelMetric( - metric_name="pass@k", - sample_level_fn=PassAtK( - sample_scoring_function=MultilingualExtractiveMatchMetric( - language=Language.ENGLISH, - gold_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], - pred_extraction_target=[IndicesExtractionConfig(prefix_for_extraction="NativeLetters")], - precision=6, - ), - ), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - prediction_perplexity = SampleLevelMetric( - metric_name="ppl", - sample_level_fn=PerplexityPreparator("words"), - category=SamplingMethod.PERPLEXITY, - corpus_level_fn=CorpusLevelPerplexityMetric("perplexity"), - higher_is_better=True, - ) - recall_at_k = SampleLevelMetric( - metric_name="recall", - sample_level_fn=Recall(k=1), - category=SamplingMethod.LOGPROBS, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - rouge_t5 = CorpusLevelMetricGrouping( - metric_name=["rouge1", "rouge2", "rougeL", "rougeLsum"], - sample_level_fn=ROUGE( - ["rouge1", "rouge2", "rougeL", "rougeLsum"], - bootstrap=True, - normalize_gold=bigbench_normalizer, - normalize_pred=bigbench_normalizer, - ), - category=SamplingMethod.GENERATIVE, - corpus_level_fn={"rouge1": np.mean, "rouge2": np.mean, "rougeL": np.mean, "rougeLsum": np.mean}, - higher_is_better={"rouge1": True, "rouge2": True, "rougeL": True, "rougeLsum": True}, - ) - rouge1 = SampleLevelMetric( - metric_name="rouge1", - sample_level_fn=ROUGE("rouge1"), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - rouge2 = SampleLevelMetric( - metric_name="rouge2", - sample_level_fn=ROUGE("rouge2"), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - rougeL = SampleLevelMetric( - metric_name="rougeL", - sample_level_fn=ROUGE("rougeL"), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - rougeLsum = SampleLevelMetric( - metric_name="rougeLsum", - sample_level_fn=ROUGE("rougeLsum"), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - simpleqa_judge = SampleLevelMetricGrouping( - metric_name=["simpleqa_judge"], - higher_is_better={"simpleqa_judge": True}, - category=SamplingMethod.GENERATIVE, - sample_level_fn=JudgeLLMSimpleQA(), - corpus_level_fn={ - "simpleqa_judge": np.mean, - }, - batched_compute=True, - ) - target_perplexity = SampleLevelMetric( - metric_name="ppl", - sample_level_fn=TargetPerplexityPreparator(units_type="words"), - category=SamplingMethod.LOGPROBS, - corpus_level_fn=CorpusLevelPerplexityMetric("perplexity"), - higher_is_better=False, - ) - ter = CorpusLevelMetric( - metric_name="ter", - sample_level_fn=GenerativePreparator(), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=CorpusLevelTranslationMetric("ter"), - higher_is_better=False, - ) - truthfulqa_mc_metrics = SampleLevelMetricGrouping( - metric_name=["truthfulqa_mc1", "truthfulqa_mc2"], - sample_level_fn=TruthfulqaMCMetrics(), - category=SamplingMethod.LOGPROBS, - corpus_level_fn={"truthfulqa_mc1": np.mean, "truthfulqa_mc2": np.mean}, - higher_is_better={"truthfulqa_mc1": True, "truthfulqa_mc2": True}, - ) - word_perplexity = CorpusLevelMetric( - metric_name="word_perplexity", - sample_level_fn=PerplexityPreparator(units_type="words"), - category=SamplingMethod.PERPLEXITY, - corpus_level_fn=CorpusLevelPerplexityMetric("weighted_perplexity"), - higher_is_better=False, - ) - gpqa_instruct_metric = SampleLevelMetric( - metric_name="extractive_match", - sample_level_fn=MultilingualExtractiveMatchMetric( - language=Language.ENGLISH, - gold_extraction_target=[ - IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True) - ], - pred_extraction_target=[ - IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True) - ], - precision=6, - ), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - gpqa_instruct_pass_at_k = SampleLevelMetric( - metric_name="gpqa_pass@k", - sample_level_fn=PassAtK( - sample_scoring_function=MultilingualExtractiveMatchMetric( - language=Language.ENGLISH, - gold_extraction_target=[ - IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True) - ], - pred_extraction_target=[ - IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True) - ], - precision=6, - ), - ), - category=SamplingMethod.GENERATIVE, - corpus_level_fn=np.mean, - higher_is_better=True, - ) - - def __str__(self): - return self.name.replace("_at_", "@") - - def __call__(self, sample_params): - # When parametrizing, we don't look at the Metrics enum, - # but at a specific single metric (a value) - # Be very careful to not change the default value of the enum - return deepcopy(self.value)(sample_params=sample_params) diff --git a/src/lighteval/metrics/metrics_corpus.py b/src/lighteval/metrics/metrics_corpus.py deleted file mode 100644 index 92c2c574a..000000000 --- a/src/lighteval/metrics/metrics_corpus.py +++ /dev/null @@ -1,192 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -"""This module manages all the metrics occurring at the corpus level. -Some metrics (such as corpus BLEU) are not computed at the individual item level, but over all the corpus. -A number of these aggregations come from the EleutherAIHarness -""" - -import logging -import math -from abc import ABC, abstractmethod -from typing import Literal - -import numpy as np -import sacrebleu -import sklearn.metrics - -from lighteval.metrics.sample_preparator import ( - GenerativeCorpusMetricInput, - LogprobCorpusMetricInput, - PerplexityCorpusMetricInput, -) -from lighteval.utils.utils import as_list - - -logger = logging.getLogger(__name__) - - -class CorpusLevelComputation(ABC): - @abstractmethod - def compute_corpus(self, items): - raise NotImplementedError - - def __str__(self): - attrs = vars(self) - attr_strs = [] - for k, v in attrs.items(): - if callable(v): - val_str = v.__name__ - else: - val_str = str(v) - attr_strs.append(f"{k}={val_str}") - return f"{self.__class__.__name__}({', '.join(attr_strs)})" - - -# General aggregations -class MatthewsCorrCoef(CorpusLevelComputation): - def compute_corpus(self, items: list[GenerativeCorpusMetricInput]) -> float: - """Computes the Matthews Correlation Coefficient, using scikit learn ([doc](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.matthews_corrcoef.html)). - - Args: - items (list[dict]): List of GenerativeCorpusMetricInput - - Returns: - float: Score - """ - golds = [i.golds for i in items] - preds = [i.preds for i in items] - return sklearn.metrics.matthews_corrcoef(golds, preds) - - -class CorpusLevelF1Score(CorpusLevelComputation): - def __init__(self, average: str, num_classes: int = 2): - """Stores the relevant parameters for the task's corpus level f1 score. - - Args: - average (str): Method to use to compute the f1 score. Can be weighted, macro, micro. - num_classes (int, optional): Num of possible choice classes. Defaults to 2. If this parameter is above 2, we'll compute multi f1 corpus score - """ - if average not in ["weighted", "macro", "micro", None]: - raise ValueError( - f"A CorpusLevelF1Score must be initialized with weighted, macro, micro, or None as an average function. {average} was used." - ) - self.average = average - self.num_classes = num_classes - - def compute_corpus(self, items: list[LogprobCorpusMetricInput]): - """Computes the metric score over all the corpus generated items, by using the scikit learn implementation.""" - golds = [i.golds for i in items] - preds = [i.preds for i in items] - # Single f1 - if self.num_classes == 2: - fscore = sklearn.metrics.f1_score(golds, preds, average=self.average) - return np.max(fscore) - - # Multi f1 - f1s = [] - for i in range(self.num_classes): - f1s.append( - sklearn.metrics.f1_score( - y_true=[g == i for g in golds], y_pred=[p == i for p in preds], average=self.average - ) - ) - return float(np.mean(f1s)) - - -class CorpusLevelTranslationMetric(CorpusLevelComputation): - def __init__(self, metric_type: str, lang: Literal["zh", "ja", "ko", ""] = ""): - """Stores the relevant parameters for a corpus level translation metric. - - Args: - metric_type (str): Can be any of bleu, chrf, or ter depending on the metric to use. - lang (str): Language code for the translation metric. - """ - self.metric_type = metric_type - self.lang = lang - - def get_metric(self): - if self.metric_type == "bleu": - import nltk - - nltk.download("punkt_tab") - return sacrebleu.BLEU(trg_lang=self.lang) - elif self.metric_type == "chrf": - return sacrebleu.CHRF() - elif self.metric_type == "chrf++": - return sacrebleu.CHRF(word_order=2) - elif self.metric_type == "ter": - return sacrebleu.TER(asian_support=True if self.lang != "" else False) - else: - raise ValueError(f"Unknown corpus level translation metric type : {self.metric_type}") - - def compute_corpus(self, items: list[GenerativeCorpusMetricInput]) -> float: - """Computes the metric score over all the corpus generated items, by using the sacrebleu implementation.""" - metric = self.get_metric() - golds = [i.golds for i in items] - preds = [] - for i in items: - pred = as_list(i.preds) - if len(pred) > 1: - logger.info( - f"Multiple predictions present, keeping only the first prediction (when computing sacrebleu.{metric.__name__})." - ) - preds.append(pred[0]) - - if self.metric_type == "bleu": - golds = [[gold[0] for gold in golds]] - - corpus_score = metric.corpus_score(hypotheses=preds, references=golds) - score = corpus_score.score - results = float(score) - return results - - -class CorpusLevelPerplexityMetric(CorpusLevelComputation): - def __init__(self, metric_type: str): - """Stores the relevant parameter for a corpus level perplexity metric. - Perplexity metrics compute more or less the same thing, which is a variation on the - average of log-probabilities over a sequence, but the normalization and processing applied - is different depending on the metric type. - Perplexity uses an exponential and no weights for the average, weighted perplexity uses an exponential - and the number of words as weights for the log-prob average, and bits per byte uses the number of bits - for normalization and divides the results by log(2). - - Args: - metric_type (str): Can be any of `perplexity`, `weighted_perplexity` or `bits_per_byte` - """ - if metric_type not in ["perplexity", "weighted_perplexity", "bits_per_byte"]: - raise ValueError(f"Unknown corpus level perplexity metric type : {metric_type}") - - self.metric_type = metric_type - - def compute_corpus(self, items: list[PerplexityCorpusMetricInput]): - """Computes the metric score over all the corpus generated items.""" - logprobs = [i.logprobs for i in items] - weights = [i.weights for i in items] - - if self.metric_type == "perplexity": - return math.exp(-np.mean(logprobs)) - if self.metric_type == "weighted_perplexity": - return math.exp(-sum(logprobs) / sum(weights)) - if self.metric_type == "bits_per_byte": - return -sum(logprobs) / sum(weights) * 1 / math.log(2) diff --git a/src/lighteval/metrics/metrics_sample.py b/src/lighteval/metrics/metrics_sample.py deleted file mode 100644 index db14b9bf6..000000000 --- a/src/lighteval/metrics/metrics_sample.py +++ /dev/null @@ -1,1456 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -"""This module manages all the metrics occurring at the sample level. The results of said metrics are then aggregated -using simple function (min, mean, max, ...) at the corpus level. Most metrics fall under this category. -""" - -import inspect -import logging -import os -from abc import ABC, abstractmethod -from typing import Callable, Literal, Union - -import nltk -import numpy as np -from huggingface_hub import HfApi -from nltk.metrics.distance import edit_distance -from nltk.tokenize import word_tokenize -from nltk.tokenize.treebank import TreebankWordTokenizer -from nltk.translate.bleu_score import sentence_bleu -from pydantic import BaseModel -from scipy.stats import hypergeom -from transformers import AutoModelForSequenceClassification, AutoTokenizer - -from lighteval.metrics.imports.bert_scorer import BERTScorer -from lighteval.metrics.imports.data_stats_metric import DataStatsMetric -from lighteval.metrics.imports.summac import SummaCZS -from lighteval.metrics.normalizations import ( - LogProbNormalization, - LogProbTokenNorm, - normalize_log_probs, - remove_braces, - remove_braces_and_strip, -) -from lighteval.metrics.utils.judge_utils import get_judge_prompt_simpleqa, process_judge_response_simpleqa -from lighteval.metrics.utils.llm_as_judge import JudgeLM -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.requests import Doc -from lighteval.utils.utils import as_list, safe_divide - - -logger = logging.getLogger(__name__) - - -class SampleLevelComputation(ABC): - @abstractmethod - def compute(self, doc: Doc, model_response: ModelResponse, **kwargs): - raise NotImplementedError - - def __str__(self): - attrs = vars(self) - attr_strs = [] - for k, v in attrs.items(): - if callable(v): - val_str = v.__name__ - else: - val_str = str(v) - attr_strs.append(f"{k}={val_str}") - return f"{self.__class__.__name__}({', '.join(attr_strs)})" - - -class ExactMatches(SampleLevelComputation): - def __init__( - self, - aggregation_function: Callable[[list[float]], float] = max, - normalize_gold: Callable[[str], str] | None = None, - normalize_pred: Callable[[str], str] | None = None, - strip_strings: bool = False, - type_exact_match: str = "full", - ): - """An exact match class. - - Args: - aggregation_function (callable, optional): How to aggregate the item results. Defaults to max. - Used if there are several golds or predictions on which scores were computed. - normalize_gold (callable, optional): Function to use to normalize the reference strings. - Defaults to None if no normalization is applied. - normalize_pred (callable, optional): Function to use to normalize the predicted strings. - Defaults to None if no normalization is applied. - strip_strings (bool, optional): Whether to strip both reference and predictions. Defaults to False. - type_exact_match (str, optional): Defines what type of match to apply (post normalization if present). - Can be any of `prefix`, `suffix` or `full`. Defaults to "full". - `prefix` checks if the prediction starts with the gold, - `suffix` if the prediction ends with the gold, - `full` if the prediction and gold are equal - """ - self.aggregation_function = aggregation_function - self.normalize_gold = normalize_gold - self.normalize_pred = normalize_pred - self.strip_strings = strip_strings - - if type_exact_match not in ["prefix", "suffix", "full"]: - # todo: we could add a set exact match - raise ValueError( - f"type_exact_match (used in parametrized_exact_match) must be one of prefix, suffix, or full. Was {type_exact_match} instead." - ) - self.type_exact_match = type_exact_match - - def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> float: - """Computes the metric over a list of golds and predictions for one single sample. - - Args: - doc (Doc): The document containing gold references. - model_response (ModelResponse): The model's response containing predictions. - **kwargs: Additional keyword arguments. - - Returns: - float: Aggregated score over the current sample's items. - """ - results = [] - # We might need to flatten golds if they are a list of lists - golds = doc.get_golds() - for gold in golds: - for pred in model_response.final_text: - results.append(self.compute_one_item(gold=gold, pred=pred)) - return self.aggregation_function(results) - - def compute_one_item( - self, - gold: str, - pred: str, - ) -> float: - """Compares two strings only. - - Args: - gold (str): One of the possible references - pred (str): One of the possible predictions - - Returns: - float: The exact match score. Will be 1 for a match, 0 otherwise. - """ - if not pred: - return 0 - - if self.strip_strings: - gold = gold.strip() - pred = pred.strip() - - if self.normalize_gold: - gold = self.normalize_gold(gold) - if self.normalize_pred: - pred = self.normalize_pred(pred) - - if self.type_exact_match == "prefix": - return 1 if pred.startswith(gold) else 0 - if self.type_exact_match == "suffix": - return 1 if pred.endswith(gold) else 0 - return 1 if gold == pred else 0 - - -class F1_score(SampleLevelComputation): - def __init__( - self, - aggregation_function: Callable[[list[float]], float] = max, - normalize_gold: Callable[[str], str] | None = None, - normalize_pred: Callable[[str], str] | None = None, - strip_strings: bool = False, - ): - """An F1 score class. F1 is computed over the bag of words of the golds and predictions. - - Args: - aggregation_function (callable, optional): How to aggregate the item results. Defaults to max. - Used if there are several golds or predictions on which scores were computed. - normalize_gold (callable, optional): Function to use to normalize the reference strings. - Defaults to None if no normalization is applied. - normalize_pred (callable, optional): Function to use to normalize the predicted strings. - Defaults to None if no normalization is applied. - strip_strings (bool, optional): Whether to strip both reference and predictions. Defaults to False. - """ - if aggregation_function is None: - aggregation_function = max - - self.aggregation_function = aggregation_function - self.normalize_gold = normalize_gold - self.normalize_pred = normalize_pred - self.strip_strings = strip_strings - - def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> float: - """Computes the metric over a list of golds and predictions for one single sample. - - Args: - doc (Doc): The document containing gold references. - model_response (ModelResponse): The model's response containing predictions. - **kwargs: Additional keyword arguments. - - Returns: - float: Aggregated score over the current sample's items. - """ - results = [] - golds = doc.get_golds() - predictions = model_response.final_text - # We might need to flatten golds if they are a list of lists - for gold in golds: - for pred in predictions: - results.append(self.compute_one_item(gold=gold, pred=pred)) - return self.aggregation_function(results) - - def compute_one_item(self, gold: str, pred: str) -> float: - """Compares two strings only. - - Args: - gold (str): One of the possible references - pred (str): One of the possible predictions - - Returns: - float: The f1 score over the bag of words, computed using nltk. - """ - if self.normalize_gold: - gold = self.normalize_gold(gold) - - if self.normalize_pred: - pred = self.normalize_pred(pred) - - gold_bow = set(gold.split()) - pred_bow = set(pred.split()) - - ret = nltk.scores.f_measure(gold_bow, pred_bow) - - if ret is None: - return 0.0 - return ret - - -class LoglikelihoodAcc(SampleLevelComputation): - def __init__(self, logprob_normalization: LogProbNormalization | None = None): - """Log likelihood accuracy class. It tests if the highest log-probability of the possible choices - is actually in the gold ones. - - Args: - logprob_normalization (LogProbNormalization | None): The normalization to apply. - """ - self.logprob_normalization = logprob_normalization - - # Solve the choices token lengths properly - def compute( - self, - doc: Doc, - model_response: ModelResponse, - **kwargs, - ) -> int: - """Computes the log likelihood accuracy: is the choice with the highest logprob in `choices_logprob` present - in the `gold_ixs`? - - Args: - doc (Doc): The document containing choices and gold indices. - model_response (ModelResponse): The model's response containing logprobs. - **kwargs: Additional keyword arguments. - - Returns: - int: The eval score: 1 if the best log-prob choice is in gold, 0 otherwise. - """ - n_choices = len(doc.choices) - choices_logprobs = model_response.logprobs[:n_choices] - unconditioned_logprobs = None - - if len(model_response.logprobs) == n_choices * 2: - unconditioned_logprobs = model_response.logprobs[n_choices : n_choices * 2] - - gold_ixs = as_list(doc.gold_index) - choices_tokens = model_response.output_tokens[:n_choices] - - normalized_log_probs = ( - normalize_log_probs( - self.logprob_normalization, - choices_logprobs, - unconditioned_logprobs, - doc.choices, - choices_tokens, - ) - if self.logprob_normalization - else choices_logprobs - ) - - best_choice = np.argmax(normalized_log_probs) - return int(best_choice in gold_ixs) - - -class NormalizedMultiChoiceProbability(SampleLevelComputation): - def __init__( - self, - log_prob_normalization: LogProbNormalization | None = None, - aggregation_function: Callable[[np.ndarray], float] = np.max, - ): - """Returns the probability of choosing the gold choice / (sum of probabilities of all choices). If multiple choices are gold, - it returns the aggregated probability (default is max). - - Args: - log_prob_normalization (LogProbNormalization | None): The normalization to apply. - aggregation_function (Callable[[np.ndarray], float]): The function to use to aggregate gold probabilities in case of multiple golds. - """ - self.log_prob_normalization = log_prob_normalization - self.aggregation_function = aggregation_function - - def compute( - self, - doc: Doc, - model_response: ModelResponse, - **kwargs, - ) -> float: - """Computes the log likelihood probability: chance of choosing the best choice. - - Args: - doc (Doc): The document containing choices and gold indices. - model_response (ModelResponse): The model's response containing logprobs. - **kwargs: Additional keyword arguments. - - Returns: - float: The probability of the best log-prob choice being a gold choice. - """ - n_choices = len(doc.choices) - choices_logprobs = model_response.logprobs[:n_choices] - unconditioned_logprobs = None - - if len(model_response.logprobs) == n_choices * 2: - unconditioned_logprobs = model_response.logprobs[n_choices : n_choices * 2] - - gold_ixs = as_list(doc.gold_index) - choices_tokens = model_response.output_tokens[:n_choices] - - normalized_log_probs = ( - normalize_log_probs( - self.log_prob_normalization, - choices_logprobs, - unconditioned_logprobs, - doc.choices, - choices_tokens, - ) - if self.log_prob_normalization - else choices_logprobs - ) - normalized_probs = np.exp(normalized_log_probs) - - normalized_probs = safe_divide(normalized_probs[gold_ixs], np.sum(normalized_probs)) - gold_idx_agg_prob = self.aggregation_function(normalized_probs) - return gold_idx_agg_prob - - -class Probability(SampleLevelComputation): - def __init__( - self, - normalization: LogProbTokenNorm | None = None, - aggregation_function: Callable[[np.ndarray], float] = np.max, - ): - """Returns the probability of choosing gold choice ignoring probabilities of other choices. If multiple choices are gold, - it returns the aggregated probability (default is max) of the gold choices. - - Args: - normalization (Normalization | None): The normalization to apply. Only Token Normalization is supported as others don't make sense. - aggregation_function (Callable[[list[float]], float]): The function to use to aggregate gold probabilities in case of multiple golds. - """ - self.log_prob_normalization = normalization - self.aggregation_function = aggregation_function - - def compute( - self, - doc: Doc, - model_response: ModelResponse, - **kwargs, - ) -> float: - """Computes the log likelihood probability: chance of choosing the best choice. - - Args: - doc (Doc): The document containing choices and gold indices. - model_response (ModelResponse): The model's response containing logprobs. - **kwargs: Additional keyword arguments. - - Returns: - float: The probability of the best log-prob choice being a gold choice. - """ - choices_tokens = model_response.output_tokens - logprobs = model_response.logprobs - reference_texts = doc.choices - - normalized_log_probs = ( - normalize_log_probs( - normalization=self.log_prob_normalization, - choices_tokens=choices_tokens, - choices_logprob=logprobs, - choices_text=reference_texts, - unconditioned_logprob=None, - ) - if self.log_prob_normalization - else logprobs - ) - probs = np.exp(normalized_log_probs) - return self.aggregation_function(probs) - - -class Recall(SampleLevelComputation): - def __init__(self, k: int) -> None: - """Recall metric class. It checks if the top `k` best choices include one of the golds or not. - - Args: - k (int): Depth level of the recall. - Recall at 1 is equivalent to a logprob accuracy without normalization. - """ - self.recall_depth = k - - def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> int: - """Computes the recall at the requested depth level: looks at the `n` best predicted choices (with the - highest log probabilities) and see if there is an actual gold among them. - - Args: - doc (Doc): The document containing choices and gold indices. - model_response (ModelResponse): The model's response containing logprobs. - **kwargs: Additional keyword arguments. - - Returns: - int: Score: 1 if one of the top level predicted choices was correct, 0 otherwise. - """ - choices_logprobs = model_response.logprobs - gold_ixs = as_list(doc.gold_index) - - if self.recall_depth == 1: - return int(np.argmax(choices_logprobs) in gold_ixs) - return int(any(ix in gold_ixs for ix in np.array(choices_logprobs).argsort()[::-1][: self.recall_depth])) - - -class MRR(SampleLevelComputation): - def __init__(self, length_normalization: bool = False): - """A mean reciprocal rank class. - - Args: - length_normalization (bool, optional): Whether to use normalization on choice length when computing the best log-probabilities. Defaults to False. - """ - self.length_normalization = length_normalization - - def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> float: - """Mean reciprocal rank. Measures the quality of a ranking of choices (ordered by correctness). - - Args: - model_response (ModelResponse): The model's response containing logprobs. - doc (Doc): The document containing choices and gold indices. - **kwargs: Additional keyword arguments. - - Returns: - float: MRR score. - """ - choices_logprobs = model_response.logprobs - gold_ixs = as_list(doc.gold_index) - choices_logprobs = model_response.logprobs - - if self.length_normalization: - choices_logprobs = [ - choice_logprob / len(choice) for choice_logprob, choice in zip(choices_logprobs, doc.choices) - ] - - ranked_choices = [sorted(choices_logprobs, reverse=True).index(choices_logprobs[gold]) for gold in gold_ixs] - return 1.0 / (min(ranked_choices) + 1) - - -class AccGoldLikelihood(SampleLevelComputation): - def compute(self, doc, model_response, **kwargs) -> int: - """Tests if at least one of predicted gold targets' argmax of logits equals the gold. - - Args: - doc: The document containing gold references. - model_response: The model's response containing argmax logits. - **kwargs: Additional keyword arguments. - - Returns: - int: 1 if at least one of the possible golds has argmax of logits == gold, 0 otherwise. - """ - return int(any(model_response.argmax_logits_eq_gold)) - - -class ROUGE(SampleLevelComputation): - ALLOWED_ROUGE_METHODS = ["rouge1", "rouge2", "rougeL", "rougeLsum"] - - def __init__( - self, - methods: str | list[str], - multiple_golds: bool = False, - bootstrap: bool = False, - normalize_gold: Callable | None = None, - normalize_pred: Callable | None = None, - aggregation_function: Callable | None = None, - tokenizer: object = None, - ): - """A ROUGE wrapper method. Relies on `rouge_scorer`. - - Args: - methods (str | list[str]): What type of ROUGE scoring to use. Can be one or any of `rouge1`, `rouge2`, `rougeL` or `rougeLsum`. - multiple_golds (bool, optional): Whether to compute ROUGE by allowing the comparison to several golds - at once, or to compute ROUGE on individual gold/prediction pairs and aggregate afterwards. Defaults to False. - bootstrap (bool, optional): Whether to use bootstrapping. Defaults to False. - aggregation_function (callable, optional): How to aggregate the item results. Defaults to max. - Used if there are several golds or predictions on which scores were computed. - normalize_gold (callable, optional): Function to use to normalize the reference strings. - Defaults to None if no normalization is applied. - normalize_pred (callable, optional): Function to use to normalize the predicted strings. - Defaults to None if no normalization is applied. - tokenizer (object, optional): An object with `tokenize` method to be used by rouge scorer. If None, rouge-scorer's - default tokenizer will be used. - """ - if aggregation_function and bootstrap: - logger.warning("Can't use both bootstrapping and an aggregation function in Rouge. Keeping bootstrap.") - self.aggregation_function = aggregation_function - if self.aggregation_function is None: - self.aggregation_function = np.mean - - self.methods = as_list(methods) - if any(method not in self.ALLOWED_ROUGE_METHODS for method in self.methods): - raise ValueError( - f"Rouge was initialised with method {methods}, which is not in {','.join(self.ALLOWED_ROUGE_METHODS)}" - ) - self.multiple_golds = multiple_golds - self.bootstrap = bootstrap - self.normalize_gold = normalize_gold - self.normalize_pred = normalize_pred - self.tokenizer = tokenizer - self.scorer = None - - def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> float | dict: - """Computes the metric(s) over a list of golds and predictions for one single sample. - - Args: - doc (Doc): The document containing gold references. - model_response (ModelResponse): The model's response containing predictions. - **kwargs: Additional keyword arguments. - - Returns: - float or dict: Aggregated score over the current sample's items. - If several rouge functions have been selected, returns a dict which maps name and scores. - """ - from rouge_score import rouge_scorer - - golds = doc.get_golds() - predictions = model_response.final_text - - if self.scorer is None: - self.scorer = rouge_scorer.RougeScorer(self.methods, tokenizer=self.tokenizer) - - # Normalize - if self.normalize_gold: - golds = [self.normalize_gold(g) for g in golds] - - if self.normalize_pred: - predictions = [self.normalize_pred(p) for p in predictions] - - if self.bootstrap: # For t5 style rouge score - scores = self._rouge_score_with_bootsrap(golds=golds, predictions=predictions) - elif self.multiple_golds: - scores = self._rouge_score_multi_golds(golds=golds, preds=predictions) - else: - scores = self._rouge_score(golds=golds, preds=predictions) - - if len(scores) == 1: - return list(scores.values())[0] - return scores - - def _rouge_score(self, golds: list[str], preds: list[str]): - scores = {m: [] for m in self.methods} - for pred in preds: - for gold in golds: - cur_scores = self.scorer.score(gold, pred) - for method in self.methods: - scores[method].append(cur_scores[method].fmeasure) - return {method: self.aggregation_function(scores[method]) for method in self.methods} - - def _rouge_score_multi_golds(self, golds: list[str], preds: list[str]): - scores = {m: [] for m in self.methods} - for pred in preds: - cur_scores = self.scorer.score_multi(golds, pred) - for method in self.methods: - scores[method].append(cur_scores[method].fmeasure) - return {method: self.aggregation_function(scores[method]) for method in self.methods} - - def _rouge_score_with_bootsrap(self, golds: list[str], predictions: list[str]): - from rouge_score import scoring - - aggregator = scoring.BootstrapAggregator() - for g, p in zip(golds, predictions): - aggregator.add_scores(self.scorer.score(g, p)) - result = aggregator.aggregate() - return {method: result[method].mid.fmeasure * 100 for method in self.methods} - - -class BertScore(SampleLevelComputation): - def __init__( - self, - normalize_gold: Callable | None = None, - normalize_pred: Callable | None = None, - ): - r"""A BERT scorer class. Relies on some called extracted from `bert-score`. By default, will use the - `microsoft/deberta-large-mnli` as scorer. For each tokenized (pred, target) pair, it computes Precision, - Recall and F1 as following: - - Precision = \sum_{t=1}^{len(pred)} \div{max(Cos.Sim.(pred_t, target))}{IDF(pred_t)} - - Recall = \sum_{t=1}^{len(target)} \div{max(Cos.Sim.(target_t, pred))}{IDF(target_t)} - - F1 = \div{Precision * Recall}{Precision + Recall} - - in which `Cos.Sim.` is the Cosine Similarity metric and `IDF(.)` represents the Inverse Document - Frequency of its input token. It defaults to 1 for all tokens and 0 for EOS and SEP tokens. - - Args: - normalize_gold (callable, optional): Function to use to normalize the reference strings. - Defaults to None if no normalization is applied. - normalize_pred (callable, optional): Function to use to normalize the predicted strings. - Defaults to None if no normalization is applied. - """ - self.bert_scorer = None - - self.normalize_gold = normalize_gold - self.normalize_pred = normalize_pred - - def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> dict[str, float]: - """Computes the prediction, recall and f1 score using the bert scorer. - - Args: - doc (Doc): The document containing gold references. - model_response (ModelResponse): The model's response containing predictions. - **kwargs: Additional keyword arguments. - - Returns: - dict: Scores over the current sample's items. - """ - golds = doc.get_golds() - predictions = model_response.final_text - - if self.bert_scorer is None: - logger.warning("The first metric computation step might be a bit longer as we need to download the model.") - # We only initialize on first compute - self.bert_scorer = BERTScorer( - model_type="microsoft/deberta-large-mnli", lang="en", rescale_with_baseline=True, num_layers=9 - ) - golds = as_list(golds) - predictions = as_list(predictions) - # Normalize - if self.normalize_gold: - golds = [self.normalize_gold(g) for g in golds] - - if self.normalize_pred: - predictions = [self.normalize_pred(p) for p in predictions] - - p, r, f = self.bert_scorer.score(predictions, golds) - return {"BERTScore-P": p[0].item(), "BERTScore-R": r[0].item(), "BERTScore-F": f[0].item()} - - -class Extractiveness(SampleLevelComputation): - def __init__( - self, - normalize_input: callable = remove_braces, - normalize_pred: callable = remove_braces_and_strip, - input_column: str = "text", - language: Literal["en", "de", "fr", "it"] = "en", - ): - """Extractiveness metric class. - - Args: - normalize_input (callable, optional): Function to normalize the input strings. - Defaults to remove_braces from lighteval.metrics.normalizations if no normalization is applied. - normalize_pred (callable, optional): Function to use to normalize the predicted strings. - Defaults to remove_braces_and_strip from lighteval.metrics.normalizations if no normalization is applied. - input_column (str): Column in the formatted_doc to use for the input. Defaults to "text". - language (str): Language ISO code for the input text. Defaults to "en". - """ - self.stats_metric = None - self.normalize_input = normalize_input - self.normalize_pred = normalize_pred - self.input_column = input_column - self.language = language - - def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> dict[str, float]: - """Compute the extractiveness of the predictions. - - This method calculates coverage, density, and compression scores for a single - prediction against the input text. - - Args: - doc (Doc): The document containing input text. - model_response (ModelResponse): The model's response containing predictions. - **kwargs: Additional keyword arguments. - - Returns: - dict[str, float]: The extractiveness scores. - """ - if self.stats_metric is None: - self.stats_metric = DataStatsMetric(language=self.language) - - inp = doc.specific[self.input_column] - prediction = model_response.final_text[0] - if self.normalize_input: - inp = self.normalize_input(inp) - if self.normalize_pred: - prediction = self.normalize_pred(prediction) - - stats = self.stats_metric.evaluate_example(prediction, inp) - return { - "summarization_coverage": stats["coverage"], - "summarization_density": stats["density"], - "summarization_compression": stats["compression"], - } - - -class Faithfulness(SampleLevelComputation): - def __init__( - self, - normalize_input: Callable = remove_braces, - normalize_pred: Callable = remove_braces_and_strip, - input_column: str = "text", - ): - """Faithfulness metric class. - - Args: - normalize_input (callable, optional): Function to normalize the input strings. - Defaults to remove_braces from lighteval.metrics.normalizations if no normalization is applied. - normalize_pred (callable, optional): Function to use to normalize the predicted strings. - Defaults to remove_braces_and_strip from lighteval.metrics.normalizations if no normalization is applied. - input_column (str): Column in the formatted_doc to use for the input. Defaults to "text". - """ - self.summac = None - self.normalize_input = normalize_input - self.normalize_pred = normalize_pred - self.input_column = input_column - - def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> dict[str, float]: - """Compute the faithfulness of the predictions. - - The SummaCZS (Summary Content Zero-Shot) model is used with configurable granularity and model variation. - - Args: - doc (Doc): The document containing input text. - model_response (ModelResponse): The model's response containing predictions. - **kwargs: Additional keyword arguments. - - Returns: - dict[str, float]: The faithfulness scores. - """ - if self.summac is None: - self.summac = SummaCZS( - granularity="sentence", model_name="vitc", imager_load_cache=False - ) # , device=device) - inp = doc.specific[self.input_column] - predictions = model_response.final_text - prediction = predictions[0] - if self.normalize_input: - inp = self.normalize_input(inp) - if self.normalize_pred: - prediction = self.normalize_pred(prediction) - return self.summac.score_one(inp, prediction)["score"] - - -class BLEURT(SampleLevelComputation): - def __init__(self): - """Creates a BLEURT scorer using a light bleurt-tiny-512 model. - For more complex use cases, could also be Elron/bleurt-base-128 - """ - self._tokenizer = None - self._model = None - - @property - def tokenizer(self): - if self._tokenizer is None: - self._tokenizer = AutoTokenizer.from_pretrained("Elron/bleurt-tiny-512") - return self._tokenizer - - @property - def model(self): - if self._model is None: - self._model = AutoModelForSequenceClassification.from_pretrained("Elron/bleurt-tiny-512") - self._model.eval() - return self._model - - def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> float: - """Uses the stored BLEURT scorer to compute the score on the current sample. - - Args: - doc (Doc): The document containing gold references. - model_response (ModelResponse): The model's response containing predictions. - **kwargs: Additional keyword arguments. - - Returns: - float: Score over the current sample's items. - """ - predictions = model_response.final_text - golds = doc.get_golds() - if len(predictions) == 1: - predictions = predictions * len(golds) - scores = self.model(**self.tokenizer(golds, predictions, return_tensors="pt"))[0].squeeze() - return scores.item() - - -class BLEU(SampleLevelComputation): - def __init__(self, n_gram: int): - """BLEU scorer class. Relies on `nltk`'s sentencebleu for scoring. - TODO: Will have to move this to sacrebleu. - - Args: - n_gram (int): Number of n_grams to use for scoring. - """ - self.n_gram = n_gram - - def compute(self, doc: Doc, model_response: ModelResponse, **kwargs): - """Computes the sentence level BLEU between the golds and each prediction, then takes the average. - - Args: - doc (Doc): The document containing gold references. - model_response (ModelResponse): The model's response containing predictions. - **kwargs: Additional keyword arguments. - - Returns: - float: Score over the current sample's items. - """ - import nltk - - nltk.download("punkt_tab") - golds = doc.get_golds() - predictions = model_response.final_text - return np.mean([self._bleu_score(golds, p) for p in predictions]) - - def _bleu_score(self, gold: list[str], pred: str): - """Computes the BLEU score between a list of golds and the current prediction. - - Args: - gold (list[str]): Reference targets. - pred (str): One of the predicted strings. - - Returns: - float: Score over the current prediction. - """ - weights = [1 if ix == self.n_gram else 0 for ix in range(1, 5)] - return sentence_bleu([word_tokenize(g) for g in gold], word_tokenize(pred), weights=weights) - - -class StringDistance(SampleLevelComputation): - def __init__( - self, - metric_types: list[str] | str, - strip_prediction: bool = True, - ): - """Contains a number of string distance and edition metrics. Relies on nltk to compute the edit distance. - - Args: - metric_types (list[str] | str): Can be one or any of `longest_common_prefix_length`, `edit_distance` or `edit_similarity`. - strip_prediction (bool, optional): Whether to strip the prediction. Defaults to True. - """ - allowed_values = ["longest_common_prefix_length", "edit_distance", "edit_similarity"] - metric_types = as_list(metric_types) - if any(metric_type not in allowed_values for metric_type in metric_types): - raise ValueError( - f"{metric_types} is not a valid value for an EditDistance metric. Possible values are {','.join(allowed_values)}." - ) - self.metric_types = metric_types - self.strip_prediction = strip_prediction - self.sample_aggregations = {"longest_common_prefix_length": max, "edit_distance": min, "edit_similarity": max} - - def compute(self, doc: Doc, model_response: ModelResponse, **kwargs): - """Computes all the requested metrics on the golds and prediction. - - Args: - doc (Doc): The document containing gold references. - model_response (ModelResponse): The model's response containing predictions. - **kwargs: Additional keyword arguments. - - Returns: - dict: The different scores computed - """ - predictions = model_response.final_text - golds = doc.get_golds() - if len(golds) > 1: - logger.warning( - "Provided more than one gold to compute a string distance metric. Just using the first one." - ) - reference = golds[0] - - result = {m: [] for m in self.metric_types} - for sequence in predictions: - if self.strip_prediction: - completion = sequence.strip() - else: - completion = sequence - - # `reference` is the entire remaining book for each instance. - # Truncate it here to be of the same length as the completion to ensure edit-distance is meaningful. - truncated_reference = reference[: len(completion)] - - completion_tokens = np.array(TreebankWordTokenizer().tokenize(completion)) - truncated_reference_tokens = np.array(TreebankWordTokenizer().tokenize(truncated_reference)) - - if "edit_distance" in self.metric_types: - result["edit_distance"].append(edit_distance(s1=completion_tokens, s2=truncated_reference_tokens)) - if "edit_similarity" in self.metric_types: - result["edit_similarity"].append( - self.edit_similarity(s1=completion_tokens, s2=truncated_reference_tokens) - ) - if "longest_common_prefix_length" in self.metric_types: - result["longest_common_prefix_length"].append( - self.longest_common_prefix_length(s1=completion_tokens, s2=truncated_reference_tokens) - ) - - final_result = {} - # We cast to float as final results can be numpy types, not JSON serializable - for m, v in result.items(): - final_result[m] = float(self.sample_aggregations[m](v)) - - return final_result - - def longest_common_prefix_length(self, s1: np.ndarray, s2: np.ndarray) -> float: - """Compute the length of the longest common prefix.""" - min_len = min(len(s1), len(s2)) - s1, s2 = s1[:min_len], s2[:min_len] - (nonzeros,) = np.cumprod(s1 == s2).nonzero() - return int(np.max(nonzeros)) + 1 if len(nonzeros) > 0 else 0 - - def edit_similarity(self, s1, s2): - """Compute the edit similarity between two lists of strings. - - Edit similarity is also used in the paper - Lee, Katherine, et al. - "Deduplicating training data makes language models better." - arXiv preprint arXiv:2107.06499 (2021). - - Returns: - float: Edit similarity score between 0 and 1 - """ - edist = edit_distance(s1, s2) - return 1.0 - edist / max(len(s1), len(s2)) if len(s1) > 0 and len(s2) > 0 else 0 - - -class JudgeLLM(SampleLevelComputation): - available_models_openai = ["gpt-3.5-turbo", "gpt-4o", "gpt-4-turbo", "gpt-4", "gpt-4o-2024-08-06"] - - def __init__( - self, - judge_model_name: str, - template: Callable, - process_judge_response: Callable, - judge_backend: Literal["litellm", "openai", "transformers", "vllm", "tgi", "inference-providers"], - short_judge_name: str | None = None, - response_format: BaseModel | None = None, - url: str | None = None, - api_key: str | None = None, - hf_provider: str | None = None, - max_tokens: int | None = None, - backend_options: dict | None = None, - ) -> None: - logger.debug(f"Initializing JudgeLLM with backend: {judge_backend}, model: {judge_model_name}") - - match judge_backend: - case "openai": - if judge_model_name not in self.available_models_openai: - raise ValueError(f"{judge_model_name} not in available models for llm as a judge metric") - api_key = api_key or os.getenv("OPENAI_API_KEY") - logger.debug("Using OpenAI backend for llm as a judge metric") - - case "tgi": - api_key = api_key or os.getenv("HF_TOKEN") - if url is None: - url = "https://api-inference.huggingface.co/v1/" - logger.debug("Using TGI backend") - - case "inference-providers": - api_key = api_key or os.getenv("HF_TOKEN") - logger.debug("Using Hugging Face Inference backend") - - case "litellm": - logger.debug("Using LiteLLM backend for llm as a judge metric") - - case "transformers" | "vllm": - logger.debug("Checking availability of Transformers or VLLM model") - api = HfApi() - models = api.list_models(model_name=judge_model_name) - if not models: - raise ValueError(f"{judge_model_name} not found on Hugging Face Hub") - - case _: - raise ValueError(f"{judge_backend} is not a valid backend for llm as a judge metric") - - self.short_judge_name = short_judge_name - self.judge = JudgeLM( - model=judge_model_name, - templates=template, - process_judge_response=process_judge_response, - judge_backend=judge_backend, - response_format=response_format, - api_key=api_key, - url=url, - hf_provider=hf_provider, - max_tokens=max_tokens, - backend_options=backend_options, - ) - - def compute(self, **kwargs) -> list: - # When deriving: Use model_responses/docs for batched eval, model_response/doc for non batched eval - raise NotImplementedError("This method should be implemented in the subclass.") - - -class JudgeLLMSimpleQA(JudgeLLM): - def __init__(self): - super().__init__( - judge_model_name="gpt-4o-2024-08-06", - template=get_judge_prompt_simpleqa, - process_judge_response=process_judge_response_simpleqa, - judge_backend="openai", - short_judge_name="gpt4o", - ) - - def compute(self, responses: list[ModelResponse], docs: list[Doc], **kwargs) -> list: - """Compute the score of a generative task using a llm as a judge. - The generative task can be multiturn with 2 turns max, in that case, we - return scores for turn 1 and 2. Also returns user_prompt and judgement - which are ignored later by the aggregator. - """ - questions = [formatted_doc.query for formatted_doc in docs] - options = [formatted_doc.choices for formatted_doc in docs] - golds = [formatted_doc.get_golds()[0] for formatted_doc in docs] - predictions = [response.final_text[0] for response in responses] - - scores, messages, judgements = self.judge.evaluate_answer_batch(questions, predictions, options, golds) - - metrics = [] - for i in range(len(docs)): - metrics.append( - { - "simpleqa_judge": scores[i], - f"user_prompt_{self.short_judge_name}": messages[i], - f"judgement_{self.short_judge_name}": judgements[i], - } - ) - - return metrics - - -class JudgeLLMMTBench(JudgeLLM): - def compute(self, model_response: list[ModelResponse], doc: list[Doc], **kwargs): - """Compute the score of a generative task using a llm as a judge. - The generative task can be multiturn with 2 turns max, in that case, we - return scores for turn 1 and 2. Also returns user_prompt and judgement - which are ignored later by the aggregator. - """ - import json - - model_responses = as_list(model_response) - docs = as_list(doc) - - # If we are evaluating a multiturn task, we need to have specific field in the formatted doc - questions = [doc.specific["multi_turn_queries"] for doc in docs] - golds = [doc.specific.get("reference", None) for doc in docs] - predictions = [response.final_text[0] for response in model_responses] - - query_context_1 = {"query": questions[0], "context": ""} - query_context_2 = {"query": questions[1], "context": predictions[0]} - - score_turn_1, message_turn_1, judgement_turn_1 = self.judge.evaluate_answer( - question=json.dumps(query_context_1, indent=2), answer=predictions[0], gold=golds[0] if golds else None - ) - score_turn_2, message_turn_2, judgement_turn_2 = self.judge.evaluate_answer( - question=json.dumps(query_context_2, indent=2), answer=predictions[1], gold=golds[1] if golds else None - ) - - return { - "judge_score_turn_1": score_turn_1, - "judge_score_turn_2": score_turn_2, - "user_prompt": [message_turn_1, message_turn_2], - "judgement": [judgement_turn_1, judgement_turn_2], - } - - -class JudgeLLMMixEval(JudgeLLM): - def compute(self, responses: list[ModelResponse], docs: list[Doc], **kwargs): - """Compute the score of a generative task using a llm as a judge. - The generative task can be multiturn with 2 turns max, in that case, we - return scores for turn 1 and 2. Also returns user_prompt and judgement - which are ignored later by the aggregator. - """ - questions = [doc.specific["question"] for doc in docs] - options = [doc.choices for doc in docs] - golds = [doc.get_golds()[0] for doc in docs] - predictions = [response.final_text[0] for response in responses] - - scores, messages, judgements = self.judge.evaluate_answer_batch(questions, predictions, options, golds) - - metrics = [] - for i in range(len(docs)): - metrics.append( - { - f"judge_score_{self.short_judge_name}": scores[i], - # f"user_prompt_{self.short_judge_name}": messages[i], - # f"judgement_{self.short_judge_name}": judgements[i], - } - ) - - return metrics - - -class SamplingMetric: - """All sampling metrics we have defined below use the same set of normalization parameters and same behavior for the default sample_scoring_function. - This class just holds the normalization and applies it to all samples passed to preprocess, then uses the default sample function if not provided. - """ - - def __init__( - self, - normalize: Callable | str | None = None, - strip_strings: bool = False, - sample_scoring_function: Callable[[Doc, ModelResponse], float] | str | None = None, - ): - if isinstance(normalize, str): - import lighteval.metrics.normalizations - - allowed_normalizations = inspect.getmembers( - lighteval.metrics.normalizations, inspect.isfunction - ) # -> {name: fn} - if normalize in allowed_normalizations: - self.normalize = allowed_normalizations[normalize] - else: - raise ValueError(f"Unknown normalization function: {normalize}") - else: - self.normalize = normalize - self.strip_strings = strip_strings - - if callable(sample_scoring_function): - self.compute_score = sample_scoring_function - self.type_exact_match = None - elif isinstance(sample_scoring_function, SampleLevelComputation): - self.compute_score = sample_scoring_function.compute - self.type_exact_match = None - else: - if isinstance(sample_scoring_function, str): - if sample_scoring_function not in ["prefix", "suffix", "full"]: - raise ValueError( - f"type_exact_match (used in parametrized_exact_match) must be one of prefix, suffix, or full. Was {sample_scoring_function} instead." - ) - self.type_exact_match = sample_scoring_function - else: - self.type_exact_match = "full" - self.compute_score = self.default_sample_scoring - - def preprocess(self, text: str) -> str: - if not text: - return "" - - if self.strip_strings: - text = text.strip() - - if self.normalize is not None: - text = self.normalize(text) - - return text - - def default_sample_scoring(self, doc: Doc, model_response: ModelResponse) -> int: - gold = doc.get_golds()[0] - pred = model_response.final_text[0] - if self.type_exact_match == "prefix": - return 1 if pred.startswith(gold) else 0 - if self.type_exact_match == "suffix": - return 1 if pred.endswith(gold) else 0 - return 1 if gold == pred else 0 - - def name_metrics(self) -> str | list[str]: - raise NotImplementedError - - -class AvgAtN(SamplingMetric, SampleLevelComputation): - def __init__(self, n: int | None = None, **kwargs): - """Sample score averages all the individual n predictions scores. - - Args: - n (int | None): Number of samples to generate - **kwargs: Additional keyword arguments. - """ - super().__init__(**kwargs) - self.n = n - self.attribute_must_be_set = ["n"] - - def compute(self, doc: Doc, model_response: ModelResponse, **kwargs): - """Computes the metric over a list of golds and predictions for one single sample. - It applies normalisation (if needed) to model prediction and gold, and takes the most frequent answer of all the available ones, - then compares it to the gold. - - Args: - model_response (ModelResponse): The model's response containing predictions. - doc (Doc): The document containing gold references. - **kwargs: Additional keyword arguments. - - Returns: - float: Aggregated score over the current sample's items. - """ - all_scores = [] - for i in range(self.n): - all_scores.append(self.compute_score(doc, model_response[i])) - - avg_score = np.mean(all_scores) - return avg_score - - def num_samples(self): - return self.n - - -class MajAtN(SamplingMetric, SampleLevelComputation): - def __init__(self, n: int | None = None, **kwargs): - """An exact match class. - - Args: - n (int): Total number of samples to generate - **kwargs: Additional keyword arguments. - """ - super().__init__(**kwargs) - - self.n = n - self.attribute_must_be_set = ["n"] - - def compute(self, doc: Doc, model_response: ModelResponse, **kwargs): - """Computes the metric over a list of golds and predictions for one single sample. - It applies normalisation (if needed) to model prediction and gold, and takes the most frequent answer of all the available ones, - then compares it to the gold. - - Args: - doc (Doc): The document containing gold references. - model_response (ModelResponse): The model's response containing predictions. - **kwargs: Additional keyword arguments. - - Returns: - float: Aggregated score over the current sample's items. - """ - if self.n is None: - raise Exception("You did not set the value of n") - - golds = doc.get_golds() - if len(golds) > 1: - raise Exception("Cannot compute maj@n with several golds") - - processed_choices = [self.preprocess(text=g) for g in doc.get_golds()] - new_doc = Doc( - choices=processed_choices, - query=doc.query, - gold_index=doc.gold_index, - ) - all_answers = [] - for pred in model_response.final_text[: self.n]: - all_answers.append(self.preprocess(text=pred)) - majority_prediction = max(all_answers, key=all_answers.count) - new_model_response = ModelResponse( - text=[majority_prediction], - ) - return self.compute_score(new_doc, new_model_response) - - def num_samples(self): - return self.n - - -class PassAtK(SamplingMetric, SampleLevelComputation): - def __init__(self, k: int | None = None, n: int | None = None, **kwargs): - """Computing pass at k with an estimator - - Args: - k (int | None): Number of correct samples threshold - n (int | None): Total number of samples to generate. - **kwargs: Additional keyword arguments. - """ - super().__init__(**kwargs) - self.k = k - self.n = n - self.attribute_must_be_set = ["k"] - - def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> float: - """Computes the metric over a list of golds and predictions for one single item with possibly many samples. - It applies normalisation (if needed) to model prediction and gold, computes their per prediction score, - then aggregates the scores over the samples using a pass@k. - - Args: - doc (Doc): The document containing gold references. - model_response (ModelResponse): The model's response containing predictions. - **kwargs: Additional keyword arguments. - - Returns: - float: Aggregated score over the current sample's items. - """ - golds = doc.get_golds() - if len(golds) > 1: - raise Exception("Cannot compute pass@k with several golds") - - predictions = model_response.final_text - if self.n is None: - self.n = len(predictions) - logger.warning("n undefined in the pass@k. We assume it's the same as the sample's number of predictions.") - elif len(predictions) < self.n: - logger.warning(f"Number of predictions is less than {self.n} for pass@k.") - - processed_choices = [self.preprocess(g) for g in doc.choices] - new_doc = Doc( - choices=processed_choices, - query=doc.query, - gold_index=doc.gold_index, - ) - - all_scores = [] - for pred in predictions[: self.n]: - cur_pred = self.preprocess(pred) - new_model_response = ModelResponse( - text=[cur_pred], - ) - all_scores.append(self.compute_score(doc=new_doc, model_response=new_model_response)) - - return self.pass_at_k(all_scores) - - def pass_at_k(self, all_scores: list[int]) -> float: - """Algo from https://arxiv.org/pdf/2107.03374""" - c: int = all_scores.count(1) - if self.n - c < self.k: - return 1.0 - - return 1.0 - np.prod(1.0 - self.k / np.arange(self.n - c + 1, self.n + 1)) - - def num_samples(self): - return self.n if self.n is not None else self.k - - -class GPassAtK(SamplingMetric, SampleLevelComputation): - def __init__( - self, - k: Union[int, list[int]] | None = None, - n: int | None = None, - thresholds: list[float] = [0.0, 0.25, 0.5, 0.75, 1.0], - name_prefix: str = None, - **kwargs, - ): - """Computing G-Pass@k from http://arxiv.org/abs/2412.13147 - - Args: - k (Union[int, list[int]] | None): Number of correct samples threshold - n (int | None): Total number of samples to generate. - thresholds (list[float]): Thresholds to control successful attempts in k generate. - name_prefix (str | None): Prefix for the metric name. - **kwargs: Additional keyword arguments. - """ - super().__init__(**kwargs) - self._k = k - self.n = n - self.attribute_must_be_set = ["k"] - - self.thresholds = thresholds - self.name = (f"{name_prefix}_" if name_prefix else "") + "g-pass@" - - @property - def k(self): - return as_list(self._k) - - @k.setter - def k(self, new_val): - self._k = as_list(new_val) - - def compute(self, doc: Doc, model_response: ModelResponse, **kwargs): - """Computes the metric over a list of golds and predictions for one single item with possibly many samples. - It applies normalisation (if needed) to model prediction and gold, computes their per prediction score, - then aggregates the scores over the samples using a pass@k. - - Args: - model_response (ModelResponse): The model's response containing predictions. - doc (Doc): The document containing gold references. - **kwargs: Additional keyword arguments. - - Returns: - float: Aggregated score over the current sample's items. - """ - golds = doc.get_golds() - predictions = model_response.final_text - - if len(golds) > 1: - raise Exception("Cannot compute G-Pass@k with several golds") - - if self.n is None: - self.n = len(predictions) - logger.warning( - "n undefined in the G-Pass@k. We assume it's the same as the sample's number of predictions." - ) - elif len(predictions) < self.n: - logger.warning(f"Number of predictions is less than {self.n} for G-Pass@k.") - - processed_choices = [self.preprocess(text=g) for g in doc.choices] - new_doc = Doc( - choices=processed_choices, - query=doc.query, - gold_index=doc.gold_index, - ) - - all_scores = [] - for pred in predictions[: self.n]: - cur_pred = self.preprocess(text=pred) - new_model_response = ModelResponse( - text=[cur_pred], - ) - all_scores.append(self.compute_score(new_doc, new_model_response)) - - return self.g_pass_at_k(all_scores) - - def g_pass_at_k(self, all_scores: list[int]) -> float: - """Computation of G-Pass@k details from http://arxiv.org/abs/2412.13147""" - c: int = sum(all_scores) - n: int = self.n - ks: int = self.k - thresholds: list[float] = self.thresholds - - def _compute_g_pass_at_k(n, c, k, m): - if m > min(c, k) or k > n or c < 0 or n <= 0 or m < 0: - return 0.0 - return hypergeom.sf(m - 1, n, c, k) - - def compute_g_pass_at_k(n, c, k, t): - m = max(int(np.ceil(k * t)), 1) - return _compute_g_pass_at_k(n, c, k, m) - - def compute_mg_pass_at_k(n, c, k): - low, high = int(np.ceil(k * 0.5)), k - - mg_pass_at_k = 0.0 - for i in range(low + 1, high + 1): - mg_pass_at_k += _compute_g_pass_at_k(n, c, k, i) - mg_pass_at_k = 2 * mg_pass_at_k / k - - return mg_pass_at_k - - metrics = {} - for k in ks: - for t in thresholds: - metrics[f"{self.name}{k}_{t}"] = compute_g_pass_at_k(n, c, k, t) - metrics[f"m{self.name}{k}"] = compute_mg_pass_at_k(n, c, k) - - return metrics - - @property - def metric_names(self): - ks: int = self.k - thresholds: list[float] = self.thresholds - - metrics = [] - for k in ks: - for t in thresholds: - metrics.append(f"{self.name}{k}_{t}") - metrics.append(f"m{self.name}{k}") - - return metrics - - def num_samples(self): - return self.n if self.n is not None else self.k diff --git a/src/lighteval/metrics/normalizations.py b/src/lighteval/metrics/normalizations.py deleted file mode 100644 index ef55681b1..000000000 --- a/src/lighteval/metrics/normalizations.py +++ /dev/null @@ -1,535 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import re -import string -import sys -import unicodedata -from dataclasses import dataclass -from typing import Callable - -from lighteval.metrics.utils.linguistic_tokenizers import get_word_tokenizer -from lighteval.utils.imports import Extra, requires -from lighteval.utils.language import Language - - -# From HELM -def helm_normalizer(text: str) -> str: - """Lower text and remove punctuation, articles and extra whitespace. - Copied from the [QuAC](http://quac.ai/) evaluation script found at - https://s3.amazonaws.com/my89public/quac/scorer.py - - Returns: - str: Normalized text with punctuation, articles removed and whitespace fixed - """ - - def remove_articles(text: str) -> str: - return re.sub(r"\b(a|an|the)\b", " ", text) - - def white_space_fix(text: str) -> str: - return " ".join(text.split()) - - def homogeneize_numbers(text: str) -> str: - """Casts text to float to test if it's a number, then casts back to string. - This allows equal numbers formatted differently (1.0 vs 1 for ex) to be considered - equal. This comes from Harness DROP - check if it causes a discrep in QuAC - """ - try: - return str(float(text)) - except ValueError: - return text - - def remove_punc(text: str) -> str: - exclude = set(string.punctuation) - return "".join(ch for ch in text if ch not in exclude) - - def lower(text: str) -> str: - return text.lower() - - def _tokenize(text): - return re.split(" |-", text) - - tokens = [white_space_fix(remove_articles(homogeneize_numbers(remove_punc(lower(t))))) for t in _tokenize(text)] - return " ".join([t for t in tokens if t != ""]).strip() - - -def harness_triviaqa_normalizer(text: str) -> str: - """Normalize text for TriviaQA evaluation. - - Returns: - str: Lowercase text with punctuation removed - """ - return text.lower().translate(str.maketrans("", "", string.punctuation)) - - -def bigbench_normalizer(text: str): - """Normalize text for BigBench evaluation. - - Returns: - str: Text with " . " replaced by ".\n" - """ - return text.replace(" . ", ".\n") - - -def remove_braces(text: str) -> str: - """Remove opening and closing braces from text. - - Returns: - str: Text with braces removed from start and end - """ - if text.startswith("{"): - text = text[1:] - if text.endswith("}"): - text = text[:-1] - return text - - -def remove_braces_and_strip(text: str) -> str: - """Remove braces and strip whitespace from text. - - Returns: - str: Text with braces removed and whitespace stripped - """ - text = text.strip() - if text.startswith("{"): - text = text[1:] - if text.endswith("}"): - text = text[:-1] - return text - - -def math_normalizer(text: str) -> str: # noqa C901 - """Source: https://github.com/hendrycks/math""" - - def _remove_boxed(text: str | None) -> str: - """Extract the text within a \\boxed{...} environment. - - Example: - >>> _remove_boxed(\\boxed{\\frac{2}{3}}) - \\frac{2}{3} - - Returns: - str: The text within the boxed environment, or empty string if extraction fails - """ - if text is None: - return "" - try: - if "\\boxed " in text: - left = "\\boxed " - assert text[: len(left)] == left - return text[len(left) :] - - left = "\\boxed{" - - assert text[: len(left)] == left - assert text[-1] == "}" - - return text[len(left) : -1] - except Exception: - return "" - - def _last_boxed_only_string(text: str) -> str | None: - """Extract the last \\boxed{...} or \\fbox{...} element from a string. - - Returns: - str | None: The last boxed string element, or None if not found - """ - idx = text.rfind("\\boxed") - if idx < 0: - idx = text.rfind("\\fbox") - if idx < 0: - return None - - i = idx - right_brace_idx = None - num_left_braces_open = 0 - while i < len(text): - if text[i] == "{": - num_left_braces_open += 1 - if text[i] == "}": - num_left_braces_open -= 1 - if num_left_braces_open == 0: - right_brace_idx = i - break - i += 1 - - if right_brace_idx is None: - retval = None - else: - retval = text[idx : right_brace_idx + 1] - - return retval - - def _fix_fracs(text: str) -> str: - """Fix the formatting of fractions in the given text. - Copied from: https://github.com/hendrycks/math/blob/357963a7f5501a6c1708cf3f3fb0cdf525642761/modeling/math_equivalence.py#L1 - - Args: - text (str): The input text. - - Returns: - str: The text with properly formatted fractions. - - Examples: - >>> _fix_fracs("\\frac12") - "\\frac{1}{2}" - >>> _fix_fracs("\\frac{3}{4}") - "\\frac{3}{4}" - >>> _fix_fracs("\\frac1{2}") - "\\frac{1}{2}" - """ - substrs = text.split("\\frac") - new_str = substrs[0] - if len(substrs) > 1: - substrs = substrs[1:] - for substr in substrs: - new_str += "\\frac" - if substr[0] == "{": - new_str += substr - else: - try: - assert len(substr) >= 2 - except AssertionError: - return text - a = substr[0] - b = substr[1] - if b != "{": - if len(substr) > 2: - post_substr = substr[2:] - new_str += "{" + a + "}{" + b + "}" + post_substr - else: - new_str += "{" + a + "}{" + b + "}" - else: - if len(substr) > 2: - post_substr = substr[2:] - new_str += "{" + a + "}" + b + post_substr - else: - new_str += "{" + a + "}" + b - text = new_str - return text - - def _fix_a_slash_b(text: str) -> str: - """Source: https://github.com/hendrycks/math - Reformat fractions formatted as a/b to \\frac{a}{b}. - - Example: - >>> _fix_a_slash_b("2/3") - \frac{2}{3} - - Returns: - str: Text with a/b fractions reformatted to LaTeX \\frac{a}{b} format - """ - if len(text.split("/")) != 2: - return text - a_str = text.split("/")[0] - b_str = text.split("/")[1] - try: - a = int(a_str) - b = int(b_str) - assert text == "{}/{}".format(a, b) - new_string = "\\frac{" + str(a) + "}{" + str(b) + "}" - return new_string - except Exception: - return text - - def _remove_right_units(text: str) -> str: - """Removes unit descriptions from LaTeX-formatted text, where units are indicated by "\\text{ }". - This function splits the text at each "\\text{ " and returns the part before the first occurrence, - effectively discarding any units and additional text following this pattern. This function also - trims any trailing whitespace left after removing units. - - Args: - text (str): The input string potentially containing LaTeX-style unit descriptions. - - Returns: - str: The text with unit descriptions removed. - - Examples: - - Input: '50.5 \\text{ kg}' - Output: '50.5' - - - Input: 'The mass is 20 grams' - Output: 'The mass is 20 grams' - - - Input: 'The object weighs 30.2 \\text{ lbs} and is 15 \\text{ inches} long' - Output: 'The object weighs 30.2' - - - Input: '\\text{ unit without preceding text}' - Output: '' - - Note: - This function assumes that "\\text{ " is only used to denote units. It will remove all text - following the first occurrence of "\\text{ ", including any further text and units that might - appear in complex sentences. - """ - # Check for "\\text{ " and split the text at each occurrence - if "\\text{ " in text: - splits = text.split("\\text{ ") - # Return only the first part which is assumed to contain the main content without units - return splits[0].rstrip() - else: - return text - - def _fix_sqrt(text: str) -> str: - """Source: https://github.com/hendrycks/math - Reformat square roots. - - Example: - >>> _fix_sqrt("\\sqrt3") - \\sqrt{3} - - Returns: - str: Text with square roots reformatted to proper LaTeX format - """ - if "\\sqrt" not in text: - return text - splits = text.split("\\sqrt") - new_string = splits[0] - for split in splits[1:]: - if split[0] != "{": - a = split[0] - new_substr = "\\sqrt{" + a + "}" + split[1:] - else: - new_substr = "\\sqrt" + split - new_string += new_substr - return new_string - - text = _remove_boxed(_last_boxed_only_string(text)) - - to_replace_1 = [ - ("\n", ""), # linebreaks - ("\\!", ""), # remove inverse spaces - ("\\\\", "\\"), # replace \\ with \ - ("tfrac", "frac"), # replace tfrac and dfrac with frac - ("dfrac", "frac"), - ("\\left", ""), # remove \left and \right - ("\\right", ""), - ("^{\\circ}", ""), # Remove circ (degrees) - ("^\\circ", ""), - ("\\$", ""), # remove dollar signs - ] - - for input_str, output_str in to_replace_1: - text = text.replace(input_str, output_str) - - # remove units (on the right) - text = _remove_right_units(text) - - to_replace_2 = [ - ("\\%", ""), # remove percentage - (r"\%", ""), - ( - " .", - " 0.", - ), # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the text - ("{.", "{0."), - ] - for input_str, output_str in to_replace_2: - text = text.replace(input_str, output_str) - - # if empty, return empty text - if len(text) == 0: - return text - if text[0] == ".": - text = "0" + text - - # to consider: get rid of e.g. "k = " or "q = " at beginning - if len(text.split("=")) == 2: - if len(text.split("=")[0]) <= 2: - text = text.split("=")[1] - - # fix sqrt3 --> sqrt{3} - text = _fix_sqrt(text) - - # remove spaces - text = text.replace(" ", "") - - # \frac1b or \frac12 --> \frac{1}{b} and \frac{1}{2}, etc. Even works with \frac1{72} (but not \frac{72}1). Also does a/b --> \\frac{a}{b} - text = _fix_fracs(text) - - # manually change 0.5 --> \frac{1}{2} - if text == "0.5": - text = "\\frac{1}{2}" - - # NOTE: X/Y changed to \frac{X}{Y} in dataset, but in simple cases fix in case the model output is X/Y - text = _fix_a_slash_b(text) - - return text - - -def gsm8k_normalizer(text: str) -> str: - """From https://github.com/openai/grade-school-math/blob/3101c7d5072418e28b9008a6636bde82a006892c/grade_school_math/dataset.py#L28 - - Args: - text (str): input text - - Returns: - str: Output text, either the number found in the text or "[invalid]" if - no number was found - """ - ANS_RE = re.compile(r"#### (\-?[0-9\.\,]+)") - INVALID_ANS = "[invalid]" - - match = ANS_RE.search(text) - if match: - match_str = match.group(1).strip() - match_str = match_str.replace(",", "") - return match_str - else: - return INVALID_ANS - - -PUNCT = {chr(i) for i in range(sys.maxunicode) if unicodedata.category(chr(i)).startswith("P")}.union( - string.punctuation -) - -_ARTICLE_PATTERNS = { - Language.ENGLISH: r"\b(a|an|the)\b", - Language.SPANISH: r"\b(el|la|los|las|un|una|unos|unas)\b", - Language.PORTUGUESE: r"\b(o|a|os|as|um|uma|uns|umas)\b", - Language.ITALIAN: r"\b(il|lo|la|i|gli|le|un|uno|una)\b", - Language.FRENCH: r"\b(le|la|les|l'|un|une|des)\b", - Language.GERMAN: r"\b(der|die|das|den|dem|des|ein|eine|einer|eines|einem|einen)\b", - Language.FINNISH: r"\b(se|yksi|yks)\b", - Language.GREEK: r"\b(ὁ|οἱ|τοῦ|τῶν|τόν|τούς|ὦ|ἡ|αἱ|τῆς|τῶν|τήν|τάς|τό|τά|τοῦ|τῶν|τό|τά)\b", - Language.NORWEGIAN: r"\b(en|ei|et|den|det|de)\b", - Language.SWEDISH: r"\b(en|ett|den|det|de)\b", - Language.TURKISH: r"\b(bir)\b", - Language.DUTCH: r"\b(de|het|een)\b", - Language.HUNGARIAN: r"\b(a|az|egy)\b", - Language.CATALAN: r"\b(el|la|els|les|un|una|uns|unes)\b", - Language.HEBREW: r"\b(ה)\b", - Language.GALICIAN: r"\b(o|a|os|as|un|unha|uns|unhas)\b", -} - - -def remove_articles(text: str, lang: Language) -> str: - """Removes definite and indefinite articles from the text. - Generated using LLM then manually checked by non-expert. - We currently only support languages that don't blend articles. - If you are a native speaker of a language where articles are blended, - we would appreciate your contribution! - - Returns: - str: Text with articles removed for the specified language - """ - pattern = _ARTICLE_PATTERNS.get(lang) - return re.sub(pattern, " ", text) if pattern else text - - -def remove_punc(text: str) -> str: - """Remove punctuation from text. - - Returns: - str: Text with punctuation removed - """ - return "".join(ch for ch in text if ch not in PUNCT) - - -@requires(Extra.MULTILINGUAL) -def get_multilingual_normalizer(lang: Language, lower: bool = True) -> Callable[[str], str]: - """Get a normalizer function for the specified language. - - Returns: - Callable[[str], str]: A function that normalizes text for the specified language - """ - - def _inner_normalizer(text: str) -> str: - tokenizer = get_word_tokenizer(lang) - text = remove_articles(text, lang) - text = remove_punc(text) - if lower: - text = text.lower() - - tokens = tokenizer.word_tokenize(text) - return " ".join(tokens) - - return _inner_normalizer - - -# Loglikelihood normalization -@dataclass -class LogProbPMINorm: - """Performs Pointwise mutual information normalization. log_likelihood_conditioned - log_likelihood_unconditioned. - Useful when answer contains generally unlikely tokens. - """ - - name: str = "norm_pmi" - - -@dataclass -class LogProbTokenNorm: - """Performs token level normalization. log_likelihood/token_length. - Useful for non-english languages. - """ - - name: str = "norm_token" - - -@dataclass -class LogProbCharNorm: - """Performs character level normalization. log_likelihood/char_length - ignore_first_space (bool, optional): Whether to ignore the first token's log prob (if it's a space only). Defaults to False. - The only case when it should be True is when the possible choices (for example `A`,`B` ...) have an extra - space added in front of them to manage tokenization issues (` A`, ` B`, ...) for some models. - """ - - name: str = "norm" - - ignore_first_space: bool = False - - -LogProbNormalization = LogProbCharNorm | LogProbTokenNorm | LogProbPMINorm - - -def normalize_log_probs( - normalization: LogProbNormalization, - choices_logprob: list[float], - unconditioned_logprob: list[float] | None, - choices_text: list[str] | None, - choices_tokens: list[list[int]] | None, -) -> list[float]: - normalized_log_probs = choices_logprob - match normalization: - case LogProbCharNorm(ignore_first_space=True): - assert choices_text is not None, "choices_text must be provided for character normalization" - if len(choices_text) != len(choices_logprob): - raise ValueError("choices_text and choices_logprob must have the same length") - normalized_log_probs = [ - choices_logprob[ix] / (len(choice) - 1 if choice[0] == " " else len(choice)) - for ix, choice in enumerate(choices_text) - ] - case LogProbCharNorm(ignore_first_space=False): - assert choices_text is not None, "choices_text must be provided for character normalization" - normalized_log_probs = [choices_logprob[ix] / len(choice) for ix, choice in enumerate(choices_text)] - case LogProbTokenNorm(): - assert choices_tokens is not None, "choices_tokens must be provided for token normalization" - normalized_log_probs = [ - choices_logprob[ix] / len(choices_tokens[ix]) for ix in range(len(choices_logprob)) - ] - case LogProbPMINorm(): - assert unconditioned_logprob is not None, "unconditioned_logprob must be provided for PMI normalization" - normalized_log_probs = [ - choices_logprob[ix] - unconditioned_logprob[ix] for ix in range(len(choices_logprob)) - ] - - return normalized_log_probs diff --git a/src/lighteval/metrics/sample_preparator.py b/src/lighteval/metrics/sample_preparator.py deleted file mode 100644 index 8d79cb448..000000000 --- a/src/lighteval/metrics/sample_preparator.py +++ /dev/null @@ -1,222 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import logging -import re -from dataclasses import asdict, dataclass - -import numpy as np - -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.requests import Doc -from lighteval.utils.utils import as_list - - -logger = logging.getLogger(__name__) - - -@dataclass -class CorpusMetricInput: - pass - - def to_dict(self): - return asdict(self) - - -@dataclass -class GenerativeCorpusMetricInput(CorpusMetricInput): - golds: list[str] - preds: list[str] - - -@dataclass -class LogprobCorpusMetricInput(CorpusMetricInput): - golds: list[int] - preds: list[float] - - -@dataclass -class PerplexityCorpusMetricInput(CorpusMetricInput): - logprobs: list[float] - weights: list[int] - - -class Preparator: - pass - - -class GenerativePreparator(Preparator): - @staticmethod - def prepare(doc: Doc, model_response: ModelResponse, **kwargs): - """Prepares an individual generative example to the format expected by metrics computed at the corpus level (aggregated). - - Args: - doc (Doc): The document containing gold references. - model_response (ModelResponse): The model's response containing predictions. - **kwargs: Additional keyword arguments. - - Returns: - GenerativeCorpusMetricInput: Stores the golds and predictions as such - """ - golds = as_list(doc.get_golds()) - predictions = model_response.final_text - return GenerativeCorpusMetricInput(golds=golds, preds=predictions) - - def __str__(self): - attrs = vars(self) - attr_strs = [] - for k, v in attrs.items(): - if callable(v): - val_str = v.__name__ - else: - val_str = str(v) - attr_strs.append(f"{k}={val_str}") - return f"{self.__class__.__name__}({', '.join(attr_strs)})" - - -class LoglikelihoodPreparator(Preparator): - def __init__(self, is_single_token: bool = False): - """Init. - - Args: - is_single_token (bool, optional): True if the preparator is used for single token loglikelihood metrics. - These metrics are computed faster, as they only compare the single token necessary. Defaults to False. - """ - self.is_single_token = is_single_token - - def prepare(self, doc: Doc, model_response: ModelResponse, **kwargs) -> LogprobCorpusMetricInput: - """Prepares an individual loglikelihood example to the format expected by metrics computed at the corpus level (aggregated). - - Args: - doc (Doc): The document containing gold indices and choices. - model_response (ModelResponse): The model's response containing logprobs. - **kwargs: Additional keyword arguments. - - Returns: - LogprobCorpusMetricInput: Stores the golds indices and the model's choice (choice with the highest logprob) - Only the first gold index is taken for a single token loglikelihood metric - """ - gold_ixs = as_list(doc.gold_index) - choices_logprob = model_response.logprobs - if self.is_single_token: - if len(gold_ixs) > 1: - logger.warning( - "The current sample has more than one gold available, which is unexpected. We selected only the first one for the corpus aggregation of the loglikelihood metric." - ) - return LogprobCorpusMetricInput(golds=gold_ixs[0], preds=np.argmax(choices_logprob)) - - return LogprobCorpusMetricInput(golds=gold_ixs, preds=np.argmax(choices_logprob)) - - -class TargetPerplexityPreparator(Preparator): - def __init__(self, units_type: str) -> None: - """Init. - - Args: - units_type (str): Basic type of text units we want to use to weight perplexity computations. - Can be `words` or `bytes` - - Raises: - ValueError: If the unit type is not words or byte, raises a ValueError - """ - if units_type not in ["words", "bytes"]: - raise ValueError("Perplexity must be computed at either the word or byte level.") - self.units_type = units_type - - def count_units(self, text: str) -> int: - """Counts the given number of unit in the input text. - - Args: - text (str): Input text - - Returns: - int: Number of units of type `self.units_type` in the input text. - """ - if self.units_type == "words": - return len(re.split(r"\s+", text)) - if self.units_type == "bytes": - return len(text.encode("utf-8")) - - def prepare(self, doc: Doc, model_response: ModelResponse, **kwargs): - """Prepares an individual perplexity example to the format expected by metrics computed at the corpus level (aggregated). - - Args: - doc (Doc): The document containing gold references. - model_response (ModelResponse): The model's response containing logprobs. - **kwargs: Additional keyword arguments. - - Returns: - PerplexityCorpusMetricInput: Stores the measured logprobs and associated text lengths, counted in the reference unit. - """ - logprobs_flat = np.sum(model_response.logprobs) - reference_text_flat = " ".join(doc.get_golds()) - return PerplexityCorpusMetricInput(logprobs=logprobs_flat, weights=self.count_units(reference_text_flat)) - - -class PerplexityPreparator(Preparator): - def __init__(self, units_type: str) -> None: - """Init. - - Args: - units_type (str): Basic type of text units we want to use to weight perplexity computations. - Can be `words` or `bytes` - - Raises: - ValueError: If the unit type is not words or byte, raises a ValueError - """ - if units_type not in ["words", "bytes"]: - raise ValueError("Perplexity must be used with either words or bytes.") - self.units_type = units_type - - def count_units(self, text: str) -> int: - """Counts the given number of unit in the input text. - - Args: - text (str): Input text - - Returns: - int: Number of units of type `self.units_type` in the input text. - """ - if self.units_type == "words": - return len(re.split(r"\s+", text)) - if self.units_type == "bytes": - return len(text.encode("utf-8")) - - def prepare(self, doc: Doc, model_response: ModelResponse, **kwargs): - """Prepares an individual perplexity example to the format expected by metrics computed at the corpus level (aggregated). - - Args: - doc (Doc): The document containing gold references. - model_response (ModelResponse): The model's response containing logprobs. - **kwargs: Additional keyword arguments. - - Returns: - PerplexityCorpusMetricInput: Stores the measured logprobs and associated text lengths, counted in the reference unit. - """ - logprobs_flat = np.sum(model_response.logprobs) - - if doc.original_query is not None: - reference_text_flat = " ".join([doc.original_query]) - else: - reference_text_flat = " ".join([doc.query]) - - return PerplexityCorpusMetricInput(logprobs=logprobs_flat, weights=self.count_units(reference_text_flat)) diff --git a/src/lighteval/metrics/utils/extractive_match_utils.py b/src/lighteval/metrics/utils/extractive_match_utils.py deleted file mode 100644 index da0d718d7..000000000 --- a/src/lighteval/metrics/utils/extractive_match_utils.py +++ /dev/null @@ -1,632 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import re -from dataclasses import dataclass, field, replace -from functools import lru_cache -from itertools import groupby -from typing import Any, Literal, Sequence - -import sympy -from sympy import Basic, FiniteSet, MatrixBase, Number -from sympy.parsing import parse_expr - -from lighteval.metrics.utils.math_comparison import should_treat_as_complex -from lighteval.tasks.requests import Doc -from lighteval.tasks.templates.utils.formulation import ChoicePrefix, get_prefix -from lighteval.tasks.templates.utils.translation_literals import TRANSLATION_LITERALS -from lighteval.utils.imports import requires -from lighteval.utils.language import Language -from lighteval.utils.timeout import timeout - - -@requires("latex2sympy2_extended") -def latex_normalization_config_default_factory(): - from latex2sympy2_extended.latex2sympy2 import NormalizationConfig - - return NormalizationConfig( - basic_latex=True, - units=True, - malformed_operators=True, - nits=True, - boxed="all", - equations=True, - ) - - -@dataclass(frozen=True) -class LatexExtractionConfig: - """Config for extracting latex from the prediction. - - Attributes: - try_extract_without_anchor (bool): Whether to try extracting latex without requiring specific anchors like "answer:" or "final answer is" - boxed_match_priority (int): Priority of the boxed match regex (-1 never, 0 first, 55 after final answer: anchor, etc...) - normalization_config (latex2sympy2_extended.latex2sympy2.NormalizationConfig): Normalization config to use for latex extraction - """ - - try_extract_without_anchor: bool = True - boxed_match_priority: int = 55 - normalization_config: Any = field(default_factory=latex_normalization_config_default_factory) - - -@dataclass(frozen=True) -class ExprExtractionConfig: - """Config for extracting mathematical expressions. - - Attributes: - try_extract_without_anchor (bool): Whether to try extracting expressions without requiring specific anchors like "answer:" or "final answer is" - """ - - try_extract_without_anchor: bool = True - - -@dataclass(frozen=True) -class IndicesExtractionConfig: - """Config for extracting choice indices. - - Attributes: - prefix_for_extraction (ChoicePrefix): The style to use for extracting choice indices (e.g. A,B,C or 1,2,3) - try_extract_without_anchor (bool): Whether to try extracting indices without requiring specific anchors like "answer:" or "final answer is". - Recommended False for indices extraction, as some indices (for example `A` which is also a word) can lead to a lot of false positives. - """ - - prefix_for_extraction: ChoicePrefix - try_extract_without_anchor: bool = False - - -ExtractionTarget = LatexExtractionConfig | ExprExtractionConfig | IndicesExtractionConfig - - -# All of the regexes are cached, to avoid repeated compiling during processing of same task -@lru_cache(maxsize=1) -def lazy_expr_regex(expr_config: ExprExtractionConfig, language: Language) -> list[tuple[re.Pattern[str], int]]: - translation_literal = TRANSLATION_LITERALS[language] - - # Basic number patterns (no LaTeX) - number_re = ( - # Format 1: Numbers with thousand separators (e.g., "1,234.56" or "1 234.56") - r"(?:" - r"(?P-?\d{1,3}(?:[ ,]\d{3})+)(?P\.\d+)?|" - # Format 2: Simple numbers with decimal point or comma (e.g., "123.45" or "123,45") - r"(?P-?\d+)(?P[.,]\d+)|" - # Format 3: Decimal part only (e.g., ".123") - r"(?P\.\d+)|" - # Format 4: Integer only (e.g., "123") - r"(?P-?\d+)" - r")(?P\s*(?:%|[Pp]ercent|\s*[Pp]ercentage|\s*[Pp]ct))?" - ) - - # Expressions such as 1/2 - operators = [r"\+", r"\-", r"\*", r"\×", r"\/", r"\^", r"\(", r"\)", r"\÷"] - operators_re = "".join(operators) - all_expr_chars = r"[\d\.\s" + operators_re + r"]" - # Expression should have at minimum at least one operator and must start with a digit - expr_re = rf"(?P-?\(?-?\d{all_expr_chars}*[{operators_re}]{all_expr_chars}+\)?)" - - # Punctuation regexes - full_stop_re = rf"[{re.escape(translation_literal.full_stop)}\.]" - comma_re = rf"[{re.escape(translation_literal.comma)}\,]" - colon_re = rf"[{re.escape(translation_literal.colon)}\:]" - space_re = rf"(?:\s|{re.escape(translation_literal.sentence_space)})" - - currency_units = re.escape("$€£¥₹₽₪₩₫฿₡₢₣₤₥₦₧₨₩₪₫₭₮₯₰₱₲₳₴₵₶₷₸₹₺₻₼₽₾₿") - expr_prefix_re = rf"(?:^|{space_re}|\=)(?:\*\*)?" - expr_suffix_re = rf"(?:\*\*)?(?:{full_stop_re}|{comma_re}|\s?{colon_re}|{space_re}|\)|\$|$)" - # Expressions must be prefixed and suffixed while, digits don't need suffix and can have currency units preceeded, this is to ensure - # That we can extract stuff like $100 or 100m2, while we don't extract XDY2K as 2 - expr_with_anchors = rf"(?:{expr_prefix_re}{expr_re}{expr_suffix_re})" - number_with_anchors = rf"(?:{expr_prefix_re}[{currency_units}]?{number_re})" - expr_or_number = rf"(?:{expr_with_anchors}|{number_with_anchors})" - regexes: list[tuple[str, int]] = [] - - # Ideally we would have translation of such concept in each language - if language == Language.ENGLISH: - final_answer_prefixed_re = rf"(?i:final answer is)\:?\s*{expr_or_number}\.?\s?I hope" - final_answer_prefixed_just_is = rf"(?i:final answer.{{0,100}}?)\s+is\:?{expr_or_number}" - regexes.append((final_answer_prefixed_re, 0)) - regexes.append((final_answer_prefixed_just_is, 50)) - - answer_prefix_re = rf"(?i:{translation_literal.answer})" - - # Match after the last equals with answer word - require the number pattern, - equals_re_colon = rf"{answer_prefix_re}\s?{colon_re}(?:.{{0,100}}=\s*|.{{0,50}}?){expr_or_number}(?!\s*=)" - equals_re = rf"{answer_prefix_re}(?:.{{0,100}}=\s*|.{{0,50}}?){expr_or_number}(?!\s*=)" - regexes.extend([(equals_re_colon, 100), (equals_re, 200)]) - - if expr_config.try_extract_without_anchor: - # If everything fails, try to match plain expr/number - regexes.append((expr_with_anchors, 300)) - regexes.append((number_with_anchors, 300)) - - return [(re.compile(pattern), priority) for pattern, priority in regexes] - - -def make_latex_env_pattern(prefix: str = "", context: Literal["boxed", "plain"] = "plain") -> str: - """Creates a LaTeX environment pattern with uniquely prefixed group names. - - Args: - prefix (str): Prefix to add to group names to make them unique - context (Literal["boxed", "plain"]): Type of content to match inside the environments - - "boxed": Match environments containing \boxed{...} - - "plain": Match any LaTeX content - - Returns: - str: Regex pattern for matching LaTeX environments with percent suffix - """ - percent_re_group = rf"(?P<{prefix}percent>(?:\\?%|[Pp]ercent|[Pp]ercentage|[Pp]ct))" - - # Define base content patterns - display_dollar_content = r"(?:[^$]|\$(?!\$))" - # Either \ not followed by ] or everything but \ - display_content_bracket = r"(?:[^\\]|\\(?!\]))" - inline_dollar_content = r"(?:\\[$]|[^\n$])" - inline_content_parenthesis = r"(?:[^\\\n]|\\(?!\)))" - inline_content_bracket = r"[^\n\]\[]" - - if context == "boxed": - # Rewrite patterns to optionally include boxed content - display_dollar_content = ( - rf"{display_dollar_content}*?\\boxed{{{display_dollar_content}+?}}{display_dollar_content}*?" - ) - display_content_bracket = ( - rf"{display_content_bracket}*?\\boxed{{{display_content_bracket}+?}}{display_content_bracket}*?" - ) - inline_dollar_content = ( - rf"{inline_dollar_content}*?\\boxed{{{inline_dollar_content}+?}}{inline_dollar_content}*?" - ) - inline_content_parenthesis = ( - rf"{inline_content_parenthesis}*?\\boxed{{{inline_content_parenthesis}+?}}{inline_content_parenthesis}*?" - ) - inline_content_bracket = ( - rf"{inline_content_bracket}*?\\boxed{{{inline_content_bracket}+?}}{inline_content_bracket}*?" - ) - else: - display_dollar_content = rf"{display_dollar_content}+?" - display_content_bracket = rf"{display_content_bracket}+?" - inline_dollar_content = rf"{inline_dollar_content}+?" - inline_content_parenthesis = rf"{inline_content_parenthesis}+?" - inline_content_bracket = rf"{inline_content_bracket}+?" - - # Build list of regex patterns - patterns = [ - # Display math environments (allow multiline) - rf"(?{display_dollar_content})(?{display_content_bracket})(?{inline_dollar_content})(?{inline_content_parenthesis})(?{inline_content_bracket})\]\s", - ] - if context == "plain": - simple_number = r"-?\d+(?:[.,]\d+)?" - patterns.append(rf"(?P<{prefix}latexFraction>-?\\frac{{{simple_number}}}{{{simple_number}}})") - - # Join patterns with | and wrap in parentheses - latex_env_re = rf"(?:(?:{'|'.join(patterns)})\s*{percent_re_group}?)" - - return latex_env_re - - -@lru_cache(maxsize=1) -def lazy_latex_regex(latex_config: LatexExtractionConfig, language: Language) -> list[tuple[re.Pattern[str], int]]: - translation_literal = TRANSLATION_LITERALS[language] - # Pattern for multiple latex environments connected by and/or - # Create patterns for up to 5 connected expressions - first_latex_group = make_latex_env_pattern("first_") - and_word = translation_literal.and_word - or_word = translation_literal.or_word - next_groups = "".join( - [rf"(?:\s*(?:{and_word}|{or_word}|,)\s*{make_latex_env_pattern(f'next{i}_')})?" for i in range(1, 6)] - ) - - latex_envs_re = rf"(?:{first_latex_group}{next_groups})" - colon_re = rf"[{re.escape(translation_literal.colon)}\:]" - answer_prefix_re = rf"(?i:{translation_literal.answer})" - - # We first match boxed env, for some reason that's the most common case of output - # Then we match the latex with environments, then we try to match the fraction - regexes: list[tuple[str, int]] = [] - for latex_re in [latex_envs_re]: - if language == Language.ENGLISH: - final_answer_prefixed_re = rf"(?i:final answer is)\:?\s*{latex_re}\.?\s?I hope" - final_answer_prefixed_just_is = rf"(?i:final answer.{{0,100}}?)\s+is\:?\s*{latex_re}" - regexes.append((final_answer_prefixed_re, 0)) - regexes.append((final_answer_prefixed_just_is, 50)) - - # Match with answer word - higher priority than plain latex - answer_re_colon = rf"{answer_prefix_re}\s?{colon_re}.{{0,50}}?{latex_re}" - answer_re = f"{answer_prefix_re}.{{0,50}}?{latex_re}" - - regexes.extend([(answer_re_colon, 100), (answer_re, 200)]) - - # Match plain LaTeX - lowest priority - if latex_config.try_extract_without_anchor: - regexes.append((latex_re, 300)) - - # This ensures that boxed is matched right after the final answer xxxx - if latex_config.boxed_match_priority >= 0: - latex_re_boxed = make_latex_env_pattern(prefix="first_", context="boxed") - next_groups = "".join( - [ - rf"(?:\s*(?:{and_word}|{or_word}|,)\s*{make_latex_env_pattern(f'next{i}_', context='boxed')})?" - for i in range(1, 6) - ] - ) - latex_re_boxed = rf"{latex_re_boxed}{next_groups}" - regexes.append((latex_re_boxed, latex_config.boxed_match_priority)) - # Match plain boxed, the issue with plain boxed is that it's impossible to know where it stops, so if there are - # till last }. We do the actuall extraction in the normalization step. - regexes.append((r"(?P\\boxed{.+})", latex_config.boxed_match_priority)) - - return [(re.compile(pattern, re.DOTALL), priority) for pattern, priority in regexes] - - -@lru_cache(maxsize=10) -def lazy_indices_regex( - indices_config: IndicesExtractionConfig, len_choices: int, language: Language -) -> list[tuple[re.Pattern[str], int]]: - translation_literal = TRANSLATION_LITERALS[language] - # First get indices to predict - indices = get_prefix(indices_config.prefix_for_extraction, translation_literal)[:len_choices] - indices_escaped = [re.escape(i) for i in indices] - # We allow both (A) and A - indices_wrapped = [rf"(?:{i}|\({i}\))" for i in indices_escaped] - indice_str_re = f"(?P{'|'.join(indices_wrapped)})" - - # The answer keys are either surrounded with **answer**., or 'answer.' or the same without the dot - full_stop_re = rf"[{re.escape(translation_literal.full_stop)}\.]" - comma_re = rf"[{re.escape(translation_literal.comma)}\,]" - colon_re = rf"[{re.escape(translation_literal.colon)}\:]" - space_re = re.escape(translation_literal.sentence_space) - - answer_prefix_re = rf"(?:^|{space_re})(?:\*\*)?" - answer_suffix_re = rf"(?:\*\*)?(?:{full_stop_re}|{comma_re}|\s?{colon_re}|{space_re}|$)" - answer_re = f"{answer_prefix_re}{indice_str_re}{answer_suffix_re}" - answer_re_start = rf"^(?:\*\*)?{indice_str_re}{answer_suffix_re}" - answer_re_line_start = rf"\n(?:\*\*)?{indice_str_re}{answer_suffix_re}" - - answer_word = f"(?i:{translation_literal.answer})" - - regexes = [] - - if language == Language.ENGLISH: - final_answer_prefixed_re = rf"(?i:final answer is)\:?\s*{indice_str_re}\.?\s?I hope" - # To allow stuff like "final answer is to your question" - final_answer_prefixed_just_is = rf"(?i:final answer.{{0,100}}?)\s+is\:?\s*{indice_str_re}" - regexes.extend( - [ - (final_answer_prefixed_re, 0), - (final_answer_prefixed_just_is, 50), - ] - ) - - regexes.extend( - [ - # Most specific patterns first - (rf"{answer_word}\s?{colon_re}.{{0,50}}?{answer_re}", 100), - # Answer word patterns - (f"{answer_word}.{{0,50}}?{answer_re}", 150), - # Start of the string - (answer_re_start, 200), - # Start of the line - (answer_re_line_start, 210), - ] - ) - - if indices_config.try_extract_without_anchor: - # Less specific patterns - regexes.extend( - [ - (answer_re, 250), - (indice_str_re, 300), - ] - ) - - return [(re.compile(pattern), priority) for pattern, priority in regexes] - - -def get_extraction_regexes_inspect( - target_types: Sequence[ExtractionTarget], language: Language, len_choices: int = 1 -) -> list[tuple[list[tuple[re.Pattern[str], int]], ExtractionTarget]]: - """Get extraction regexes for inspect AI. - Temporary implementation. - TODO: refacto this function to share code with get_extraction_regexes - """ - extraction_regexes: list[tuple[list[tuple[re.Pattern[str], int]], ExtractionTarget]] = [ - (lazy_latex_regex(target_type, language), target_type) - if isinstance(target_type, LatexExtractionConfig) - else (lazy_expr_regex(target_type, language), target_type) - if isinstance(target_type, ExprExtractionConfig) - else (lazy_indices_regex(target_type, len_choices, language), target_type) - for target_type in target_types - ] - - # Sort the extraction res so that order is indices, latex, expr - def get_target_type_order(target_type: ExtractionTarget) -> int: - match target_type: - case IndicesExtractionConfig(): - return 0 - case LatexExtractionConfig(): - return 1 - case ExprExtractionConfig(): - return 2 - - extraction_regexes = sorted(extraction_regexes, key=lambda x: get_target_type_order(x[1])) - - return extraction_regexes - - -def get_extraction_regexes( - formatted_doc: Doc, target_types: Sequence[ExtractionTarget], language: Language -) -> list[tuple[list[tuple[re.Pattern[str], int]], ExtractionTarget]]: - extraction_regexes: list[tuple[list[tuple[re.Pattern[str], int]], ExtractionTarget]] = [ - (lazy_latex_regex(target_type, language), target_type) - if isinstance(target_type, LatexExtractionConfig) - else (lazy_expr_regex(target_type, language), target_type) - if isinstance(target_type, ExprExtractionConfig) - else (lazy_indices_regex(target_type, len(formatted_doc.choices), language), target_type) - for target_type in target_types - ] - - # Sort the extraction res so that order is indices, latex, expr - def get_target_type_order(target_type: ExtractionTarget) -> int: - match target_type: - case IndicesExtractionConfig(): - return 0 - case LatexExtractionConfig(): - return 1 - case ExprExtractionConfig(): - return 2 - - extraction_regexes = sorted(extraction_regexes, key=lambda x: get_target_type_order(x[1])) - - return extraction_regexes - - -# Small cache, to catche repeated calls invalid parsing -@lru_cache(maxsize=20) -@requires("latex2sympy2_extended") -def parse_latex_with_timeout(latex: str, timeout_seconds: int): - from latex2sympy2_extended.latex2sympy2 import latex2sympy - - return timeout(timeout_seconds)(latex2sympy)( - latex, is_real=not should_treat_as_complex(latex), convert_degrees=False, normalization_config=None - ) - - -@lru_cache(maxsize=20) -def parse_expr_with_timeout(expr: str, timeout_seconds: int): - return timeout(timeout_seconds)(parse_expr)(expr, evaluate=False) - - -def extract_expr(match: re.Match, timeout_seconds: int) -> tuple[str | sympy.Expr | None, str]: - # First combine the number - groups = match.groupdict() - # Expr group will always exist because every regex has it - expr = groups.get("expr", "") - integer = next((val for name, val in groups.items() if name.startswith("integer") and val), "") - decimal = next((val for name, val in groups.items() if name.startswith("decimal") and val), "") - - is_percentage = True if groups.get("percent", None) else False - - if integer or decimal: - # This makes sure we can convert numbers like 0001 to 1. Do note that this can convert 0 to '', so we assume an empty string was 0 and convert it back afterwards. - integer = integer.translate(str.maketrans("", "", ", ")).lstrip("0") - if len(integer) == 0: - integer = "0" - - decimal = decimal.replace(",", ".") - number_str = f"{integer}{decimal}" - try: - number = Number(number_str) - except Exception: - return None, number_str - - if is_percentage: - number = convert_to_pct(number) - return number, number_str - - # Otherwise just return the expression - # Remove new lines and spaces - if expr: - try: - return parse_expr_with_timeout(expr.replace("\n", " ").replace("^", "**"), timeout_seconds), expr - except Exception: # noqa: E722 - pass - return None, expr - - -def convert_to_pct(number: Number): - return sympy.Mul(number, sympy.Rational(1, 100), evaluate=False) - - -@requires("latex2sympy2_extended") -@lru_cache(maxsize=20) -def extract_latex( - match: re.Match, latex_config: LatexExtractionConfig, timeout_seconds: int -) -> tuple[sympy.Expr | str | None, str]: - from latex2sympy2_extended.latex2sympy2 import FiniteSet as L2SFiniteSet - from latex2sympy2_extended.latex2sympy2 import normalize_latex - - latex_exprs = [] - latex_strs = [] - - # Get all latex groups (both first_ and nextN_ prefixes) - first_latex_group = next( - ((val, name) for name, val in match.groupdict().items() if name.startswith("first_latex") and val), None - ) - - # Get all nextN_ groups - next_latex_groups = [ - next( - ((val, name) for name, val in match.groupdict().items() if name.startswith(f"next{i}_latex") and val), None - ) - for i in range(1, 6) - ] - - all_latex = list(filter(lambda x: x is not None, [first_latex_group] + next_latex_groups)) - - for latex, name in all_latex: - name_without_prefix = name.split("_")[0] - group_name = name.split("_")[1] if len(name.split("_")) > 1 else None - is_percentage = True if match.groupdict().get(f"{name_without_prefix}_percent") else False - - # Use modified config if group name is 'boxed' - config = latex_config.normalization_config - if group_name == "latexBoxed": - config = replace(config, boxed="last") # Use replace to modify single field - - normalized_latex = normalize_latex( - latex, - config=config, - ) - latex_strs.append(normalized_latex) - - try: - parsed_latex = parse_latex_with_timeout(normalized_latex, timeout_seconds=timeout_seconds) - if is_percentage: - parsed_latex = convert_to_pct(parsed_latex) - latex_exprs.append(parsed_latex) - except Exception: # noqa: E722 - latex_exprs.append(None) - pass - - if not latex_exprs: - return None, "" - - # If we have multiple expressions and all of them are parsed, wrap them in a Tuple - if len(latex_exprs) > 1 and all(expr is not None for expr in latex_exprs): - # To handle solution is: 1,2 and 3 - all_elements = [] - for expr in latex_exprs: - if isinstance(expr, FiniteSet): - all_elements.extend(expr.args) - else: - all_elements.append(expr) - return L2SFiniteSet(*all_elements), " and ".join(latex_strs) - - # Otherwise return the single expression - return latex_exprs[0], latex_strs[0] - - -def extract_indices( - match: re.Match, target_type: IndicesExtractionConfig, timeout_seconds: int -) -> tuple[str | None, str]: - def normalize_index(index: str) -> str: - return index.replace("(", "").replace(")", "").strip() - - return normalize_index(match.group("indices")), normalize_index(match.group("indices")) - - -def extract_match( - match: re.Match, target_type: ExtractionTarget, timeout_seconds: int -) -> tuple[Basic | MatrixBase | str | None, str]: - """Extracts the match from the regex match. - - Args: - match (re.Match): The regex match object containing the extracted text - target_type (ExtractionTarget): The type of extraction to perform (latex, expression, or indices) - timeout_seconds (int): Maximum time in seconds to spend parsing expressions - - Returns: - tuple[Basic | MatrixBase | str | None, str]: A tuple containing: - - The extracted and parsed value (if successful) or None (if parsing failed) - - The string representation of the extracted text - """ - if isinstance(target_type, LatexExtractionConfig): - return extract_latex(match, target_type, timeout_seconds=timeout_seconds) - elif isinstance(target_type, ExprExtractionConfig): - return extract_expr(match, timeout_seconds=timeout_seconds) - elif isinstance(target_type, IndicesExtractionConfig): - return extract_indices(match, target_type, timeout_seconds=timeout_seconds) - - -def extract_target_from_pred( - pred: str, - target_res: list[tuple[list[tuple[re.Pattern[str], int]], ExtractionTarget]], - fallback_mode: Literal["no_fallback", "first_match"] = "no_fallback", - extraction_mode: Literal["first_match", "any_match"] = "any_match", - timeout_seconds: int = 5, -): - """Extracts targets from a prediction string using regex patterns. - Returns first sucesffuly extracted match. - - Args: - pred (str): The prediction string to extract from - target_res (list[tuple[list[tuple[re.Pattern[str], int]], ExtractionTarget]]): List of regex patterns and their priorities for each target type - fallback_mode (Literal["no_fallback", "first_match"], optional): How to handle extraction failures. Defaults to "no_fallback". - - "no_fallback": Return only successfully parsed match - - "first_match": Additionaly Include the first string match no matter how parsing finished - extraction_mode (Literal["first_match", "any_match"], optional): How to handle extraction failures. Defaults to "any_match". - - "first_match": Only tries to extract the first match - - "any_match": Tries to extract any match - timeout_seconds (int, optional): Maximum time in seconds to spend parsing each expression. Defaults to 5. - - Returns: - list: List of extracted predictions, with first fallbac string appended if fallback_mode is "first_match" - """ - extracted_predictions = [] - fallbacks = [] - - # Get all patterns and sort by priority - all_patterns = [ - (pattern, target_type, priority) - for target_patterns, target_type in target_res - for pattern, priority in target_patterns - ] - match_found = False - - # Group patterns by priority using itertools.groupby - for _, patterns_group in groupby(sorted(all_patterns, key=lambda x: x[2]), key=lambda x: x[2]): - # Find all matches for each pattern in this priority group - matches_with_pos = ( - (match, match.start(), match.end(), target_type) - for pattern, target_type, _ in patterns_group - for match in pattern.finditer(pred) - ) - - # Sort matches by end position (rightmost first) and then by start position (leftmost first) - matches_with_pos = sorted(matches_with_pos, key=lambda x: (x[2], -x[1]), reverse=True) - - # Try to extract from each match, starting from rightmost - for match, _, _, target_type in matches_with_pos: - extracted_match, str_fallback = extract_match(match, target_type, timeout_seconds) - match_found = True - - if str_fallback: - fallbacks.append(str_fallback) - - if extracted_match is not None: - extracted_predictions.append(extracted_match) - break - - if extraction_mode == "first_match": - break - - # If we found something and we're in first_match mode, stop processing other priorities - if extracted_predictions or (match_found and extraction_mode == "first_match"): - break - - if fallback_mode == "first_match" and fallbacks: - extracted_predictions += [fallbacks[0]] - - return extracted_predictions diff --git a/src/lighteval/metrics/utils/judge_prompts.jsonl b/src/lighteval/metrics/utils/judge_prompts.jsonl deleted file mode 100644 index a43ef34c1..000000000 --- a/src/lighteval/metrics/utils/judge_prompts.jsonl +++ /dev/null @@ -1,8 +0,0 @@ -{"name": "pair-v2", "type": "pairwise", "system_prompt": "Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user question displayed below. You should choose the assistant that follows the user's instructions and answers the user's question better. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of their responses. Begin your evaluation by comparing the two responses and provide a short explanation. Avoid any position biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. After providing your explanation, output your final verdict by strictly following this format: \"[[A]]\" if assistant A is better, \"[[B]]\" if assistant B is better, and \"[[C]]\" for a tie.", "prompt_template": "[User Question]\n{question}\n\n[The Start of Assistant A's Answer]\n{answer_a}\n[The End of Assistant A's Answer]\n\n[The Start of Assistant B's Answer]\n{answer_b}\n[The End of Assistant B's Answer]", "description": "Prompt for general questions", "category": "general", "output_format": "[[A]]"} -{"name": "pair-v2-multi-turn", "type": "pairwise", "system_prompt": "Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user questions. You should choose the assistant that follows the user's instructions and answers the user's questions better. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of their responses. You should focus on who provides a better answer to the second user question. Begin your evaluation by comparing the responses of the two assistants and provide a short explanation. Avoid any position biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. After providing your explanation, output your final verdict by strictly following this format: \"[[A]]\" if assistant A is better, \"[[B]]\" if assistant B is better, and \"[[C]]\" for a tie.", "prompt_template": "<|The Start of Assistant A's Conversation with User|>\n\n### User:\n{question_1}\n\n### Assistant A:\n{answer_a_1}\n\n### User:\n{question_2}\n\n### Assistant A:\n{answer_a_2}\n\n<|The End of Assistant A's Conversation with User|>\n\n\n<|The Start of Assistant B's Conversation with User|>\n\n### User:\n{question_1}\n\n### Assistant B:\n{answer_b_1}\n\n### User:\n{question_2}\n\n### Assistant B:\n{answer_b_2}\n\n<|The End of Assistant B's Conversation with User|>", "description": "Prompt for multi-turn general questions", "category": "general", "output_format": "[[A]]"} -{"name": "pair-math-v1", "type": "pairwise", "system_prompt": "Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user question displayed below. Your evaluation should consider correctness and helpfulness. You will be given a reference answer, assistant A's answer, and assistant B's answer. Your job is to evaluate which assistant's answer is better. Begin your evaluation by comparing both assistants' answers with the reference answer. Identify and correct any mistakes. Avoid any position biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. After providing your explanation, output your final verdict by strictly following this format: \"[[A]]\" if assistant A is better, \"[[B]]\" if assistant B is better, and \"[[C]]\" for a tie.", "prompt_template": "[User Question]\n{question}\n\n[The Start of Reference Answer]\n{ref_answer_1}\n[The End of Reference Answer]\n\n[The Start of Assistant A's Answer]\n{answer_a}\n[The End of Assistant A's Answer]\n\n[The Start of Assistant B's Answer]\n{answer_b}\n[The End of Assistant B's Answer]", "description": "Prompt for math questions", "category": "math", "output_format": "[[A]]"} -{"name": "pair-math-v1-multi-turn", "type": "pairwise", "system_prompt": "Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user questions. Your evaluation should consider correctness and helpfulness. You will be given reference answers, the assistant A's answers, the assistant B's answers. Your job is to determine which assistant provides correct and helpful answers to the second user question. Begin your evaluation by comparing both assistants' answers with the reference answers. Identify and correct any mistakes. Avoid any position biases and ensure that the order in which the responses were presented does not influence your decision. Do not allow the length of the responses to influence your evaluation. Do not favor certain names of the assistants. Be as objective as possible. After providing your explanation, output your final verdict by strictly following this format: \"[[A]]\" if assistant A is better, \"[[B]]\" if assistant B is better, and \"[[C]]\" for a tie.", "prompt_template": "<|The Start of Reference Answer|>\n\n### User:\n{question_1}\n\n### Reference answer:\n{ref_answer_1}\n\n### User:\n{question_2}\n\n### Reference answer:\n{ref_answer_2}\n\n<|The End of Reference Answer|>\n\n\n<|The Start of Assistant A's Conversation with User|>\n\n### User:\n{question_1}\n\n### Assistant A:\n{answer_a_1}\n\n### User:\n{question_2}\n\n### Assistant A:\n{answer_a_2}\n\n<|The End of Assistant A's Conversation with User|>\n\n\n<|The Start of Assistant B's Conversation with User|>\n\n### User:\n{question_1}\n\n### Assistant B:\n{answer_b_1}\n\n### User:\n{question_2}\n\n### Assistant B:\n{answer_b_2}\n\n<|The End of Assistant B's Conversation with User|>", "description": "Prompt for multi-turn general questions", "category": "general", "output_format": "[[A]]"} -{"name": "single-v1", "type": "single", "system_prompt": "You are a helpful assistant.", "prompt_template": "[Instruction]\nPlease act as an impartial judge and evaluate the quality of the response provided by an AI assistant to the user question displayed below. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of the response. Begin your evaluation by providing a short explanation. Be as objective as possible. After providing your explanation, you must rate the response on a scale of 1 to 10 by strictly following this format: \"[[rating]]\", for example: \"Rating: [[5]]\".\n\n[Question]\n{question}\n\n[The Start of Assistant's Answer]\n{answer}\n[The End of Assistant's Answer]", "description": "Prompt for general questions", "category": "general", "output_format": "[[rating]]"} -{"name": "single-math-v1", "type": "single", "system_prompt": "You are a helpful assistant.", "prompt_template": "[Instruction]\nPlease act as an impartial judge and evaluate the quality of the response provided by an AI assistant to the user question displayed below. Your evaluation should consider correctness and helpfulness. You will be given a reference answer and the assistant's answer. Begin your evaluation by comparing the assistant's answer with the reference answer. Identify and correct any mistakes. Be as objective as possible. After providing your explanation, you must rate the response on a scale of 1 to 10 by strictly following this format: \"[[rating]]\", for example: \"Rating: [[5]]\".\n\n[Question]\n{question}\n\n[The Start of Reference Answer]\n{ref_answer_1}\n[The End of Reference Answer]\n\n[The Start of Assistant's Answer]\n{answer}\n[The End of Assistant's Answer]", "description": "Prompt for general questions", "category": "math", "output_format": "[[rating]]"} -{"name": "single-v1-multi-turn", "type": "single", "system_prompt": "Please act as an impartial judge and evaluate the quality of the response provided by an AI assistant to the user question displayed below. Your evaluation should consider factors such as the helpfulness, relevance, accuracy, depth, creativity, and level of detail of the response. Your evaluation should focus on the assistant's answer to the second user question. Begin your evaluation by providing a short explanation. Be as objective as possible. After providing your explanation, you must rate the response on a scale of 1 to 10 by strictly following this format: \"[[rating]]\", for example: \"Rating: [[5]]\".\n\n", "prompt_template": "<|The Start of Assistant A's Conversation with User|>\n\n### User:\n{question_1}\n\n### Assistant A:\n{answer_1}\n\n### User:\n{question_2}\n\n### Assistant A:\n{answer_2}\n\n<|The End of Assistant A's Conversation with User|>", "description": "Prompt for general questions", "category": "general", "output_format": "[[rating]]"} -{"name": "single-math-v1-multi-turn", "type": "single", "system_prompt": "Please act as an impartial judge and evaluate the quality of the response provided by an AI assistant to the user question. Your evaluation should consider correctness and helpfulness. You will be given a reference answer and the assistant's answer. Your evaluation should focus on the assistant's answer to the second question. Begin your evaluation by comparing the assistant's answer with the reference answer. Identify and correct any mistakes. Be as objective as possible. After providing your explanation, you must rate the response on a scale of 1 to 10 by strictly following this format: \"[[rating]]\", for example: \"Rating: [[5]]\".\n\n", "prompt_template": "<|The Start of Reference Answer|>\n\n### User:\n{question_1}\n\n### Reference answer:\n{ref_answer_1}\n\n### User:\n{question_2}\n\n### Reference answer:\n{ref_answer_2}\n\n<|The End of Reference Answer|>\n\n\n<|The Start of Assistant A's Conversation with User|>\n\n### User:\n{question_1}\n\n### Assistant A:\n{answer_1}\n\n### User:\n{question_2}\n\n### Assistant A:\n{answer_2}\n\n<|The End of Assistant A's Conversation with User|>", "description": "Prompt for general questions", "category": "math", "output_format": "[[rating]]"} diff --git a/src/lighteval/metrics/utils/judge_utils.py b/src/lighteval/metrics/utils/judge_utils.py deleted file mode 100644 index cde25fd26..000000000 --- a/src/lighteval/metrics/utils/judge_utils.py +++ /dev/null @@ -1,127 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -import logging - - -logger = logging.getLogger(__name__) - - -def get_judge_prompt_simpleqa(question: str, answer: str, gold: str, **kwargs): - GRADER_TEMPLATE = """ -Your job is to look at a question, a gold target, and a predicted answer, and then assign a grade of either ["CORRECT", "INCORRECT", "NOT_ATTEMPTED"]. -First, I will give examples of each grade, and then you will grade a new example. - - -The following are examples of CORRECT predicted answers. -``` -Question: What are the names of Barack Obama's children? -Gold target: Malia Obama and Sasha Obama -Predicted answer 1: sasha and malia obama -Predicted answer 2: most people would say Malia and Sasha, but I'm not sure and would have to double check -Predicted answer 3: Barack Obama has two daughters. Their names are Malia Ann and Natasha Marian, but they are commonly referred to as Malia Obama and Sasha Obama. Malia was born on July 4, 1998, and Sasha was born on June 10, 2001. -``` -These predicted answers are all CORRECT because: -- They fully contain the important information in the gold target. -- They do not contain any information that contradicts the gold target. -- Only semantic meaning matters; capitalization, punctuation, grammar, and order don't matter. -- Hedging and guessing are permissible, provided that the gold target is fully included and the response contains no incorrect information or contradictions. - - -The following are examples of INCORRECT predicted answers. -``` -Question: What are the names of Barack Obama's children? -Gold target: Malia and Sasha -Predicted answer 1: Malia. -Predicted answer 2: Malia, Sasha, and Susan. -Predicted answer 3: Barack Obama does not have any children. -Predicted answer 4: I think it's either Malia and Sasha. Or it could be Malia and Jackie. Or it could be Joey and Malia. -Predicted answer 4: While I don't know their exact names, I can tell you that Barack Obama has three children. -Predicted answer 5: It's possible you may mean Betsy and Olivia. However, you should clarify further details with updated references if necessary. Is that the correct answer? -Predicted answer 6: It may be the case that Obama's child is named James. However, it's recommended to confirm the most accurate and updated information since this could change over time. This model may not always reflect the most current information. -``` -These predicted answers are all INCORRECT because: -- A factual statement in the answer contradicts the gold target. Incorrect statements that have some hedging (e.g., "it is possible that", "although i'm not sure, i think") are also considered incorrect. - - -The following are examples of NOT_ATTEMPTED predicted answers. -``` -Question: What are the names of Barack Obama's children? -Gold target: Malia and Sasha -Predicted answer 1: I don't know. -Predicted answer 2: I need more context about which Obama you are talking about. -Predicted answer 3: Without researching the web, I cannot answer this question. However, I can tell you that Barack Obama has two children. -Predicted answer 4: Barack Obama has two children. I know that one of them is Malia, but I'm not sure about the other one. -``` -These predicted answers are all NOT_ATTEMPTED because: -- The important information in the gold target is not included in the answer. -- No statements in the answer contradict the gold target. - - -Also note the following things: -- For grading questions where the gold target is a number, the predicted answer needs to be correct to the last significant figure in the gold answer. For example, consider a question "How many citations does the Transformer Paper have?" with gold target "120k". -- Predicted answers "120k", "124k", and 115k" are all CORRECT. -- Predicted answers "100k" and "113k" are INCORRECT. -- Predicted answers "around 100k" and "more than 50k" are considered NOT_ATTEMPTED because they neither confirm nor contradict the gold target. -- The gold target may contain more information than the question. In such cases, the predicted answer only needs to contain the information that is in the question. -- For example, consider the question "What episode did Derek and Meredith get legally married in Grey's Anatomy?" with gold target "Season 7, Episode 20: White Wedding". Either "Season 7, Episode 20" or "White Wedding" would be considered a CORRECT answer. -- Do not punish predicted answers if they omit information that would be clearly inferred from the question. -- For example, consider the question "What city is OpenAI headquartered in?" and the gold target "San Francisco, California". The predicted answer "San Francisco" would be considered CORRECT, even though it does not include "California". -- Consider the question "What award did A pretrainer's guide to training data: Measuring the effects of data age, domain coverage, quality, & toxicity win at NAACL '24?", the gold target is "Outstanding Paper Award". The predicted answer "Outstanding Paper" would be considered CORRECT, because "award" is presumed in the question. -- For the question "What is the height of Jason Wei in meters?", the gold target is "1.73 m". The predicted answer "1.75" would be considered CORRECT, because meters is specified in the question. -- For the question "What is the name of Barack Obama's wife?", the gold target is "Michelle Obama". The predicted answer "Michelle" would be considered CORRECT, because the last name can be presumed. -- Do not punish for typos in people's name if it's clearly the same name. -- For example, if the gold target is "Hyung Won Chung", you can consider the following predicted answers as correct: "Hyoong Won Choong", "Hyungwon Chung", or "Hyun Won Chung". - - -Here is a new example. Simply reply with either CORRECT, INCORRECT, NOT ATTEMPTED. Don't apologize or correct yourself if there was a mistake; we are just trying to grade the answer. -``` -Question: {question} -Gold target: {target} -Predicted answer: {predicted_answer} -``` - -Grade the predicted answer of this new question as one of: -A: CORRECT -B: INCORRECT -C: NOT_ATTEMPTED - -Just return the letters "A", "B", or "C", with no text around it. -""".strip() - - content = GRADER_TEMPLATE.format( - question=question, - target=gold, - predicted_answer=answer, - ) - return [{"role": "user", "content": content}] - - -def process_judge_response_simpleqa(response: str) -> float: - if response == "A": - return 1.0 - elif response == "B": - return 0.0 - elif response == "C": - return 0.0 - else: - logger.warning(f"Unknown response from judge: {response}") - return 0.0 diff --git a/src/lighteval/metrics/utils/linguistic_tokenizers.py b/src/lighteval/metrics/utils/linguistic_tokenizers.py deleted file mode 100644 index a5670a268..000000000 --- a/src/lighteval/metrics/utils/linguistic_tokenizers.py +++ /dev/null @@ -1,274 +0,0 @@ -# Copyright 2024 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import logging -from abc import ABC, abstractmethod -from functools import lru_cache -from typing import Callable, Iterator - -from lighteval.utils.imports import ( - Extra, - requires, -) -from lighteval.utils.language import Language - - -logger = logging.getLogger(__name__) - - -# Copy of https://github.com/huggingface/datatrove/blob/main/src/datatrove/utils/tokenization.py -def strip_strings(els: list[str]) -> list[str]: - return [el.strip() for el in els if len(el.strip()) > 0] - - -def simple_span_tokenize(text: str, sents: list[str]) -> Iterator[tuple[int, int]]: - start_index = 0 - for sent in sents: - start_char = text.index(sent, start_index) - end_char = start_char + len(sent) - start_index = end_char - yield start_char, end_char - - -class WordTokenizer(ABC): - @abstractmethod - def word_tokenize(self, text: str) -> list[str]: - pass - - @abstractmethod - def sent_tokenize(self, text: str) -> list[str]: - pass - - @abstractmethod - def span_tokenize(self, text: str) -> list[tuple[int, int]]: - pass - - -class WhitespaceTokenizer(WordTokenizer): - def word_tokenize(self, text: str) -> list[str]: - return text.split() - - def sent_tokenize(self, text: str) -> list[str]: - return text.split() - - def span_tokenize(self, text: str) -> list[tuple[int, int]]: - return list(simple_span_tokenize(text, text.split())) - - -class NLTKTokenizer(WordTokenizer): - def __init__(self, punkt_language: str): - super().__init__() - self.punkt_language = punkt_language - self._tokenizer = None - - @property - def tokenizer(self): - if self._tokenizer is None: - from nltk import download, load - - download("punkt_tab") - self._tokenizer = load(f"tokenizers/punkt/{self.punkt_language}.pickle") - return self._tokenizer - - def word_tokenize(self, text) -> list[str]: - from nltk.tokenize import word_tokenize - - tokens = word_tokenize(text, language=self.punkt_language) - return strip_strings(tokens) - - def sent_tokenize(self, text: str) -> list[str]: - from nltk.tokenize import sent_tokenize - - sents = sent_tokenize(text, language=self.punkt_language) - return strip_strings(sents) - - def span_tokenize(self, text: str) -> list[tuple[int, int]]: - return list(self.tokenizer.span_tokenize(text)) - - -@requires(Extra.MULTILINGUAL) -class SpaCyTokenizer(WordTokenizer): - def __init__(self, spacy_language: str, config=None): - super().__init__() - self.spacy_language = spacy_language - self.config = config - self._tokenizer = None - - @property - def tokenizer(self): - if self._tokenizer is None: - import spacy - - if self.config is None: - self._tokenizer = spacy.blank(self.spacy_language) - else: - self._tokenizer = spacy.blank(self.spacy_language, config=self.config) - self._tokenizer.add_pipe("sentencizer") - return self._tokenizer - - def word_tokenize(self, text: str) -> list[str]: - self.tokenizer.max_length = len(text) + 10 - tokens = [token.text for token in self.tokenizer(text, disable=["parser", "tagger", "ner"])] - return strip_strings(tokens) - - def sent_tokenize(self, text: str) -> list[str]: - self.tokenizer.max_length = len(text) + 10 - sents = [sent.text for sent in self.tokenizer(text, disable=["parser", "tagger", "ner"]).sents] - return strip_strings(sents) - - def span_tokenize(self, text: str) -> list[tuple[int, int]]: - return [ - (sent.start_char, sent.end_char) - for sent in self.tokenizer(text, disable=["parser", "tagger", "ner"]).sents - ] - - -@requires("stanza") -class StanzaTokenizer(WordTokenizer): - def __init__(self, stanza_language: str, **stanza_kwargs): - super().__init__() - self.stanza_language = stanza_language - self.stanza_kwargs = stanza_kwargs - self._tokenizer = None - - @property - def tokenizer(self): - if self._tokenizer is None: - import stanza - from stanza.pipeline.core import DownloadMethod - - self._tokenizer = stanza.Pipeline( - self.stanza_language, - processors="tokenize", - download_method=DownloadMethod.REUSE_RESOURCES, - **self.stanza_kwargs, - ) - - return self._tokenizer - - def word_tokenize(self, text: str) -> list[str]: - doc = self.tokenizer(text) - tokens = [token.text for sentence in doc.sentences for token in sentence.tokens] - return strip_strings(tokens) - - def sent_tokenize(self, text: str) -> list[str]: - doc = self.tokenizer(text) - sents = [sentence.text for sentence in doc.sentences] - return strip_strings(sents) - - def span_tokenize(self, text: str) -> list[tuple[int, int]]: - doc = self.tokenizer(text) - return [(sent.tokens[0].start_char, sent.tokens[-1].end_char) for sent in doc.sentences] - - -# If you know a better tokenizer or better proxy language, please submit a PR -TOKENIZER_FACTORY: dict[Language, Callable[[], WordTokenizer]] = { - Language.ENGLISH: lambda: SpaCyTokenizer("en"), - Language.KOREAN: lambda: SpaCyTokenizer("ko"), - Language.GERMAN: lambda: SpaCyTokenizer("de"), - Language.FRENCH: lambda: SpaCyTokenizer("fr"), - Language.CZECH: lambda: SpaCyTokenizer("cz"), - Language.DANISH: lambda: SpaCyTokenizer("da"), - Language.DUTCH: lambda: SpaCyTokenizer("nl"), - Language.ESTONIAN: lambda: SpaCyTokenizer("et"), - Language.FINNISH: lambda: SpaCyTokenizer("fi"), - Language.GREEK: lambda: SpaCyTokenizer("el"), - Language.ITALIAN: lambda: SpaCyTokenizer("it"), - Language.MALAYALAM: lambda: SpaCyTokenizer("ml"), - Language.NORWEGIAN: lambda: SpaCyTokenizer("nb"), - Language.POLISH: lambda: SpaCyTokenizer("pl"), - Language.PORTUGUESE: lambda: SpaCyTokenizer("pt"), - Language.RUSSIAN: lambda: SpaCyTokenizer("ru"), - Language.SLOVENIAN: lambda: SpaCyTokenizer("sl"), - Language.SPANISH: lambda: SpaCyTokenizer("es"), - Language.SWEDISH: lambda: SpaCyTokenizer("sv"), - Language.TURKISH: lambda: SpaCyTokenizer("tr"), - Language.CHINESE: lambda: SpaCyTokenizer("zh", {"nlp": {"tokenizer": {"segmenter": "jieba"}}}), - Language.JAPANESE: lambda: SpaCyTokenizer("ja"), # note that there are some issues for >50k chars text - Language.VIETNAMESE: lambda: SpaCyTokenizer("vi"), - Language.INDONESIAN: lambda: SpaCyTokenizer("id"), - Language.PERSIAN: lambda: SpaCyTokenizer("fa"), - Language.ARABIC: lambda: SpaCyTokenizer("ar"), - Language.HINDI: lambda: SpaCyTokenizer("hi"), - Language.TAMIL: lambda: SpaCyTokenizer("ta"), - Language.URDU: lambda: SpaCyTokenizer("ur"), - Language.MARATHI: lambda: SpaCyTokenizer("mr"), - Language.TELUGU: lambda: SpaCyTokenizer("te"), - Language.HUNGARIAN: lambda: SpaCyTokenizer("hu"), - Language.ROMANIAN: lambda: SpaCyTokenizer("ro"), - Language.UKRAINIAN: lambda: SpaCyTokenizer("uk"), - Language.SLOVAK: lambda: SpaCyTokenizer("sk"), - Language.BULGARIAN: lambda: SpaCyTokenizer("bg"), - Language.CATALAN: lambda: SpaCyTokenizer("ca"), - Language.CROATIAN: lambda: SpaCyTokenizer("hr"), - Language.LATIN: lambda: SpaCyTokenizer("la"), - Language.SERBIAN: lambda: SpaCyTokenizer("sr"), - Language.LITHUANIAN: lambda: SpaCyTokenizer("lt"), - Language.HEBREW: lambda: SpaCyTokenizer("he"), - Language.LATVIAN: lambda: SpaCyTokenizer("lv"), - Language.ICELANDIC: lambda: SpaCyTokenizer("is"), - Language.ARMENIAN: lambda: SpaCyTokenizer("hy"), - Language.BASQUE: lambda: SpaCyTokenizer("eu"), - Language.THAI: lambda: SpaCyTokenizer("th"), - Language.TAGALOG: lambda: SpaCyTokenizer("tl"), - Language.ALBANIAN: lambda: SpaCyTokenizer("sq"), - Language.MACEDONIAN: lambda: SpaCyTokenizer("mk"), - Language.AZERBAIJANI: lambda: SpaCyTokenizer("az"), - Language.AMHARIC: lambda: SpaCyTokenizer("am"), - Language.BENGALI: lambda: SpaCyTokenizer("bn"), - Language.MALAY: lambda: SpaCyTokenizer("ms"), - Language.NEPALI: lambda: SpaCyTokenizer("ne"), - Language.KAZAKH: lambda: StanzaTokenizer("kk"), - Language.GUJARATI: lambda: SpaCyTokenizer("gu"), - Language.KANNADA: lambda: SpaCyTokenizer("kn"), - Language.WELSH: lambda: StanzaTokenizer("cy"), - Language.NORWEGIAN_NYNORSK: lambda: SpaCyTokenizer("nn"), - Language.SINHALA: lambda: SpaCyTokenizer("si"), - Language.TATAR: lambda: SpaCyTokenizer("tt"), - Language.AFRIKAANS: lambda: SpaCyTokenizer("af"), - Language.KIRGHIZ: lambda: SpaCyTokenizer("ky"), - Language.IRISH: lambda: SpaCyTokenizer("ga"), - Language.LUXEMBOURGISH: lambda: SpaCyTokenizer("lb"), - Language.MALTESE: lambda: StanzaTokenizer("mt"), - Language.SANSKRIT: lambda: SpaCyTokenizer("sa"), - Language.YORUBA: lambda: SpaCyTokenizer("yo"), - Language.SERBOCROATIAN: lambda: SpaCyTokenizer("sr"), - Language.BELARUSIAN: lambda: StanzaTokenizer("be"), - # proxies - Language.BOSNIAN: lambda: SpaCyTokenizer("hr"), # Proxy - Language.ESPERANTO: lambda: SpaCyTokenizer("en"), # Proxy - Language.OCCITAN: lambda: SpaCyTokenizer("ca"), # Proxy - Language.SWAHILI: lambda: SpaCyTokenizer("en"), # Proxy - Language.JAVANESE: lambda: SpaCyTokenizer("en"), # Proxy - Language.UZBEK: lambda: SpaCyTokenizer("tr"), # Proxy, alternative ru - Language.TAJIK: lambda: SpaCyTokenizer("ru"), # Proxy - Language.KURDISH: lambda: SpaCyTokenizer("en"), # Proxy, multiple scripts! - Language.BASHKIR: lambda: SpaCyTokenizer("tt"), # Proxy - Language.WESTERN_FRISIAN: lambda: SpaCyTokenizer("nl"), # Proxy - Language.BRETON: lambda: StanzaTokenizer("cy"), # Proxy - Language.MALAGASY: lambda: SpaCyTokenizer("en"), # Proxy - Language.YIDDISH: lambda: SpaCyTokenizer("he"), # Proxy - Language.SOMALI: lambda: SpaCyTokenizer("en"), # Proxy - Language.TURKMEN: lambda: SpaCyTokenizer("tr"), # Proxy - Language.PASHTO: lambda: SpaCyTokenizer("xx"), # Proxy. xx is "multi-language" -} - - -@lru_cache -def get_word_tokenizer(language: Language) -> WordTokenizer: - tokenizer = TOKENIZER_FACTORY.get(language) - if tokenizer is None: - logger.warning(f"No word tokenizer found for language {language}, will split on spaces.") - return WhitespaceTokenizer() - return tokenizer() diff --git a/src/lighteval/metrics/utils/llm_as_judge.py b/src/lighteval/metrics/utils/llm_as_judge.py deleted file mode 100644 index 4a68a1ed3..000000000 --- a/src/lighteval/metrics/utils/llm_as_judge.py +++ /dev/null @@ -1,449 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -import asyncio -import logging -import time -from concurrent.futures import ThreadPoolExecutor -from dataclasses import dataclass -from typing import Callable, Literal, Optional - -from huggingface_hub import AsyncInferenceClient, InferenceTimeoutError -from pydantic import BaseModel -from requests.exceptions import HTTPError -from tqdm import tqdm -from tqdm.asyncio import tqdm_asyncio - -from lighteval.utils.imports import raise_if_package_not_available -from lighteval.utils.utils import as_list - - -logging.getLogger("openai").setLevel(logging.ERROR) -logging.getLogger("httpx").setLevel(logging.ERROR) -logger = logging.getLogger(__name__) - - -DEFAULT_FORMAT = {"type": "text"} - - -@dataclass -class LitellmBackendOptions: - """Options for the LiteLLM judge backend with default values. - - Attributes: - caching (bool): Whether to enable caching for the API responses. Defaults to True. - concurrent_requests (int): The maximum number of concurrent requests to the API. Defaults to 10. - increase_max_tokens_for_reasoning (bool): Whether to increase the max tokens for certain reasoning - models. Defaults to True. - """ - - caching: bool = True - concurrent_requests: int = 10 - - # Increases max_tokens depending on the model used, see implementation below - increase_max_tokens_for_reasoning: bool = True - - -class JudgeLM: - """A class representing a judge for evaluating answers using either the chosen backend. - - Attributes: - model (str): The name of the model. - templates (Callable): A function taking into account the question, options, answer, and gold and returning the judge prompt. - process_judge_response (Callable): A function for processing the judge's response. - judge_backend (Literal["litellm", "openai", "transformers", "tgi", "vllm", "inference-providers"]): The backend for the judge. - url (str | None): The URL for the OpenAI API. - api_key (str | None): The API key for the OpenAI API (either OpenAI or HF key). - max_tokens (int): The maximum number of tokens to generate. Defaults to 512. - response_format (BaseModel | None): The format of the response from the API, used for the OpenAI and TGI backend. - hf_provider (Literal["black-forest-labs", "cerebras", "cohere", "fal-ai", "fireworks-ai", - "inference-providers", "hyperbolic", "nebius", "novita", "openai", "replicate", "sambanova", "together"] | None): - The HuggingFace provider when using the inference-providers backend. - backend_options (dict | None): Options for the backend. Currently only supported for litellm. - - Methods: - evaluate_answer: Evaluates an answer using the OpenAI API or Transformers library. - __lazy_load_client: Lazy loads the OpenAI client or Transformers pipeline. - __call_api: Calls the API to get the judge's response. - __call_transformers: Calls the Transformers pipeline to get the judge's response. - __call_vllm: Calls the VLLM pipeline to get the judge's response. - """ - - def __init__( - self, - model: str, - templates: Callable, - process_judge_response: Callable, - judge_backend: Literal["litellm", "openai", "transformers", "tgi", "vllm", "inference-providers"], - url: str | None = None, - api_key: str | None = None, - max_tokens: int | None = None, - response_format: BaseModel = None, - hf_provider: Optional[ - Literal[ - "black-forest-labs", - "cerebras", - "cohere", - "fal-ai", - "fireworks-ai", - "inference-providers", - "hyperbolic", - "nebius", - "novita", - "openai", - "replicate", - "sambanova", - "together", - ] - ] = None, - backend_options: dict | None = None, - ): - self.model = model - self.template = templates - - self.API_MAX_RETRY = 3 - self.API_RETRY_SLEEP = 1 - - self.client = None - self.pipe = None - self.process_judge_response = process_judge_response - - self.url = url - self.api_key = api_key - self.backend = judge_backend - self.hf_provider = hf_provider - self.max_tokens = max_tokens - - self.response_format = response_format if not None else DEFAULT_FORMAT - - self.backend_options = backend_options or {} - - # Override backend options dictionary with the corresponding dataclass to ensure all specified options are valid - if judge_backend == "litellm": - self.backend_options = LitellmBackendOptions(**self.backend_options) - - # Validate that hf_provider is specified when using inference-providers backend - if self.backend == "inference-providers" and self.hf_provider is None: - raise ValueError("When using 'inference-providers' as backend, you must specify an 'hf_provider'") - - def __lazy_load_client(self): # noqa: C901 - match self.backend: - # Both "openai" and "tgi" backends use the OpenAI-compatible API - # They are handled separately to allow for backend-specific validation and setup - case "openai" | "tgi": - raise_if_package_not_available("openai") - if self.client is None: - from openai import OpenAI - - self.client = OpenAI( - api_key=self.api_key if self.url is None else None, base_url=self.url if self.url else None - ) - return self.__call_api_parallel - - case "litellm": - raise_if_package_not_available("litellm") - return self.__call_litellm - - case "vllm": - raise_if_package_not_available("vllm") - if self.pipe is None: - from vllm import LLM, SamplingParams - from vllm.transformers_utils.tokenizer import get_tokenizer - - self.sampling_params = SamplingParams(temperature=0.8, top_p=0.95, max_tokens=self.max_tokens) - self.tokenizer = get_tokenizer(self.model, tokenizer_mode="auto") - self.pipe = LLM(model=self.model, gpu_memory_utilization=0.8, dtype="float16") - return self.__call_vllm - - case "transformers": - if self.pipe is None: - import torch - from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline - - transformers_model = AutoModelForCausalLM.from_pretrained( - self.model, torch_dtype=torch.float16, trust_remote_code=False, device_map="cuda" - ) - tokenizer = AutoTokenizer.from_pretrained(self.model) - self.pipe = pipeline( - "text-generation", - model=transformers_model, - tokenizer=tokenizer, - max_new_tokens=self.max_tokens, - ) - return self.__call_transformers - - case "inference-providers": - from huggingface_hub import AsyncInferenceClient - - self.client = AsyncInferenceClient(token=self.api_key, base_url=self.url, provider=self.hf_provider) - return self.__call_hf_inference_async - - case _: - raise ValueError(f"Unsupported backend: {self.backend}") - - def dict_of_lists_to_list_of_dicts(self, dict_of_lists): - """Transform a dictionary of lists into a list of dictionaries. - - Each dictionary in the output list will contain one element from each list in the input dictionary, - with the same keys as the input dictionary. - - Args: - dict_of_lists: A dictionary where each value is a list. - All lists are expected to have the same length. - - Returns: - A list of dictionaries. - - Example: - >>> dict_of_lists_to_list_of_dicts({'k': [1, 2, 3], 'k2': ['a', 'b', 'c']}) - [{'k': 1, 'k2': 'a'}, {'k': 2, 'k2': 'b'}, {'k': 3, 'k2': 'c'}] - """ - # Check if input is empty - if not dict_of_lists: - return None - - # Get all list lengths to ensure they match - list_lengths = [len(values) for values in dict_of_lists.values()] - - # Ensure all lists have the same length - if len(set(list_lengths)) > 1: - raise ValueError("All lists in the input dictionary must have the same length") - - # Get the length of the lists - n = list_lengths[0] if list_lengths else 0 - - # Create list of dictionaries - result = [] - for i in range(n): - new_dict = {key: values[i] for key, values in dict_of_lists.items()} - result.append(new_dict) - - return result - - def evaluate_answer_batch( - self, - questions: list[str], - answers: list[str], - options: list[list[str]] | list[None], - golds: list[str] | list[None], - **kwargs, - ): - judge_function = self.__lazy_load_client() - - kwargss = self.dict_of_lists_to_list_of_dicts(kwargs) - if kwargss is None: - kwargss = [{} for _ in range(len(questions))] - - # enumerate over questions answers options and golds to make the - prompts = [ - self.template(question=q, answer=a, options=o, gold=g, **k) - for q, a, o, g, k in zip(questions, answers, options, golds, kwargss) - ] - responses = judge_function(prompts) - scores = [self.process_judge_response(response) for response in responses] - - # clean up the vllm pipeline and free up memory - if self.pipe is not None and self.backend == "vllm": - del self.pipe - self.pipe = None - - return scores, prompts, responses - - def evaluate_answer(self, question: str, answer: str, options: list[str] | None = None, gold: str | None = None): - """Evaluates an answer using either Transformers or OpenAI API. - - Args: - question (str): The prompt asked to the evaluated model. - answer (str): Answer given by the evaluated model. - options (list[str] | None): Optional list of answer options. - gold (str | None): Optional reference answer. - - Returns: - A tuple containing the score, prompts, and judgment. - """ - # lazy loading of the pipeline - judge_function = self.__lazy_load_client() - prompt = self.template(question=question, options=options, answer=answer, gold=gold) - response = judge_function(prompt) - score = self.process_judge_response(response) - - return score, prompt, response - - def __call_transformers(self, prompt): - response = self.pipe(prompt)[0]["generated_text"] - response = response[-1]["content"] - return response - - def __call_vllm(self, prompt): - tokenized = [self.tokenizer.apply_chat_template(p) for p in prompt] - output = self.pipe.generate(prompt_token_ids=tokenized, sampling_params=self.sampling_params, use_tqdm=True) - outputs = [output.outputs[0].text for output in output] - return outputs - - def __call_litellm(self, prompts): # noqa: C901 - import litellm - - if self.backend_options.caching: - from litellm.caching.caching import Cache, LiteLLMCacheType - - litellm.cache = Cache(type=LiteLLMCacheType.DISK) - - # Automatically drop parameters that are not supported by the currently used inference API - litellm.drop_params = True - - def __call_api(prompt): - error_message = "ERROR: Failed to get response from the API." - for _ in range(self.API_MAX_RETRY): - try: - max_new_tokens = self.max_tokens - - is_reasoning_model = "o1" in self.model or "o3" in self.model or "R1" in self.model - if is_reasoning_model and self.backend_options.increase_max_tokens_for_reasoning: - max_new_tokens = min(max_new_tokens * 10, 32000) - - kwargs = { - "model": self.model, - "messages": prompt, - "n": 1, - "caching": True, - "response_format": self.response_format, - } - if max_new_tokens is not None: - kwargs["max_tokens"] = (max_new_tokens,) - if self.api_key is not None: - kwargs["api_key"] = self.api_key - if self.url is not None: - kwargs["base_url"] = self.url - - response = litellm.completion(**kwargs) - text = response.choices[0].message.content - if not text or text == error_message: - kwargs["caching"] = False - response = litellm.completion(**kwargs) - text = response.choices[0].message.content - if not text or text == error_message: - # Just return an error response if the second attempt fails too - logger.error(f"Failed to get response from the API for prompt: {prompt}") - return error_message - return text - except Exception as e: - logger.warning(f"{type(e), e}") - time.sleep(self.API_RETRY_SLEEP) - return error_message - - results = [] - with ThreadPoolExecutor(self.backend_options.concurrent_requests) as executor: - for entry in tqdm(executor.map(__call_api, prompts), total=len(prompts)): - results.append(entry) - - if None in results: - raise ValueError("Some entries are not annotated due to errors in annotate_p, please inspect and retry.") - - return results - - def __call_hf_inference_async(self, prompts): - async def run_all() -> list[str]: - """Wrap inference call into function""" - tasks = (self.__call_hf_inference(prompt) for prompt in prompts) - return await tqdm_asyncio.gather(*tasks, desc="HF inference", total=len(prompts)) - - try: - loop = asyncio.get_running_loop() - logger.debug("Exting event loop is found, using loop.create_task") - result = loop.run_until_complete(run_all()) - except RuntimeError: - logger.debug("No running event loop found, using asyncio.run") - result = asyncio.run(run_all()) - - if None in result: - logger.warning("None found in inference results") - - return result - - async def __call_hf_inference(self, prompt): - self.client: AsyncInferenceClient - for _ in range(self.API_MAX_RETRY): - try: - response = await self.client.chat_completion( - model=self.model, - messages=prompt, - max_tokens=self.max_tokens, - ) - return response.choices[0].message.content - except (InferenceTimeoutError, HTTPError) as e: - logger.warning(f"HTTP error during HF inference: {e}") - await asyncio.sleep(self.API_RETRY_SLEEP) - except Exception as e: - logger.warning(f"Unexpected error during HF inference: {e}") - await asyncio.sleep(self.API_RETRY_SLEEP) - - raise Exception("Failed to get response from the HF API") - - def __call_api_parallel(self, prompts): - results = [] - with ThreadPoolExecutor(10) as executor: - for entry in tqdm(executor.map(self.__call_api, prompts), total=len(prompts)): - results.append(entry) - - if None in results: - raise ValueError("Some entries are not annotated due to errors in annotate_p, please inspect and retry.") - - return results - - def __call_api(self, prompt): - for _ in range(self.API_MAX_RETRY): - try: - # Base model - response = self.client.beta.chat.completions.parse( - model=self.model, - messages=as_list(prompt), - response_format=self.response_format, - max_tokens=self.max_tokens, - temperature=0.0, - n=1, - ) - answer = response.choices[0].message.parsed - return answer - except TypeError: - try: - # Finetune - response = self.client.chat.completions.create( - model=self.model, - messages=as_list(prompt), - response_format=self.response_format, - max_tokens=self.max_tokens, - n=1, - ) - text = response.choices[0].message.content - return text - except Exception as e: - logger.warning(f"{type(e), e}") - time.sleep(self.API_RETRY_SLEEP) - except Exception as e: - logger.warning(f"{type(e), e}") - time.sleep(self.API_RETRY_SLEEP) - - raise Exception("Failed to get response from the API") - - def __str__(self) -> str: - return f"Model: {self.model}, Judge Backend: {self.backend}, URL: {self.url}" diff --git a/src/lighteval/metrics/utils/math_comparison.py b/src/lighteval/metrics/utils/math_comparison.py deleted file mode 100644 index 974d6d2cc..000000000 --- a/src/lighteval/metrics/utils/math_comparison.py +++ /dev/null @@ -1,611 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -# Heavily inspired by https://github.com/QwenLM/Qwen2.5-Math and https://github.com/huggingface/lm-evaluation-harness -import re -from itertools import product - -from sympy import ( - And, - Basic, - E, - Eq, - FiniteSet, - Float, - GreaterThan, - Interval, - LessThan, - MatrixBase, - MatrixExpr, - Mul, - Number, - Rational, - Set, - StrictGreaterThan, - StrictLessThan, - Symbol, - Tuple, - default_sort_key, - ordered, - simplify, -) -from sympy.core.function import UndefinedFunction -from sympy.core.relational import Relational - -from lighteval.utils.imports import requires -from lighteval.utils.timeout import timeout - - -def safe_sympy_doit(a: Basic | MatrixBase): - """Safely execute doit() on a sympy expression, catching exceptions. - Doit in sympy will evaluate expressions it will pass the expression tree and evluate nodes. - For example for 1+1+1 it will evaluate the additions and return 3. One issue with it is that it maybe - evaluates too much as integrals will also be evaluated. - - As we are using latex2sympy2_extended, evaluates are - - Args: - a: A sympy Basic or MatrixBase expression to evaluate - - Returns: - The result of a.doit() if successful, otherwise returns the original expression - """ - try: - return a.doit() - except TimeoutError: - raise - except Exception: # noqa: E722 - pass - return a - - -def is_atomic_or_pct_atomic(expr: Basic | MatrixBase, atomic_type: type) -> bool: - """Check if expression is either an atomic type or percentage atomic type. - - Args: - expr: The sympy expression to check - atomic_type: The atomic type to check for - - Returns: - True if expr is atomic_type or percentage atomic type, False otherwise - """ - return isinstance(expr, atomic_type) or ( - # Check for percentage representation: latex2sympy_extended converts "X%" into X*Rational(1,100) - # So we detect percentages by looking for this multiplication structure - isinstance(expr, Mul) - and len(expr.args) == 2 - and expr.args[1] == Rational(1, 100) - and isinstance(expr.args[0], atomic_type) - ) - - -def sympy_numeric_eq(a: Basic | MatrixBase, b: Basic | MatrixBase, precision: int): - """Compare two sympy expressions numerically with given precision. - - Args: - a: First sympy expression - b: Second sympy expression - precision: Number of decimal places to compare - - Returns: - True if expressions are numerically equal within precision, False otherwise - """ - # Only do this when one of the two is a float, in other cases use symbolic equality as this could lead to false positives - # E.g we want 1/3 == 0.333333 to work - if isinstance(a, (MatrixBase, MatrixExpr)) and isinstance(b, (MatrixBase, MatrixExpr)): - a = safe_sympy_doit(a) - b = safe_sympy_doit(b) - - # If we have matrices and one of them is only made of floats, we can use the same logic as above - if isinstance(a, (MatrixBase)) and isinstance(b, (MatrixBase)) and a.shape == b.shape: - return all(sympy_numeric_eq(a_elem, b_elem, precision) for a_elem, b_elem in zip(a.flat(), b.flat())) - - # Ensure this also works for percentage numbers so that 0.333333% = 0.33333333333 with precision 4 - elif is_atomic_or_pct_atomic(a, Number) or is_atomic_or_pct_atomic(b, Number): - # If one of them is a float or a negative atomic number, we can try to use precision - if is_atomic_or_pct_atomic(a, Float) or is_atomic_or_pct_atomic(b, Float): - a = safe_sympy_doit(a) - b = safe_sympy_doit(b) - # Now if both are numbers, we can use precision - if isinstance(a, (Number)) and isinstance(b, (Number)): - return a.round(precision) == b.round(precision) - else: - return safe_sympy_doit(a) == safe_sympy_doit(b) - - else: - try: - return (a - b).evalf(chop=True) == 0 # type: ignore - except TimeoutError: - raise - except Exception: # noqa: E722 - pass - - return False - - -def sympy_symbolic_eq(a: Basic | MatrixBase, b: Basic | MatrixBase) -> bool: - """Compare two sympy expressions symbolically. - - Args: - a: First sympy expression - b: Second sympy expression - - Returns: - True if expressions are symbolically equal, False otherwise - """ - try: - a_b_diff = simplify((a - b)) # type: ignore - if isinstance(a_b_diff, MatrixBase) and a_b_diff.is_zero_matrix: - return True - elif isinstance(a_b_diff, Basic) and a_b_diff.is_zero: - return True - except TimeoutError: - raise - except Exception: # noqa: E722 - pass - - return False - - -def sympy_deep_compare_set_and_tuple(gold: FiniteSet | Tuple, pred: FiniteSet | Tuple, precision: int) -> bool: - """Compare two finite sets by comparing each element with given precision. - - Args: - gold (FiniteSet | Tuple): First finite set or tuple. - pred (FiniteSet | Tuple): Second finite set or tuple. - precision (int): Number of decimal places to compare. - - Returns: - True if sets contain equal elements within precision, False otherwise. - - Note: in order to fully support finite sets, we should ideally do kartesian product comparison - but this is not implemented yet. We kinda hope sympy will order the elements. - """ - from latex2sympy2_extended.sets import FiniteSet as L2SFiniteSet - - def unwrap_eq(s): - if is_assignment_relation(s): - return take_last_relation(s).rhs - return s - - def sort_key(x): - try: - return default_sort_key(unwrap_eq(x).evalf()) - except TimeoutError: - raise - except Exception: # noqa: E722 - return default_sort_key(unwrap_eq(x)) - - # This ensures it works for {1/3} and {0.333333} - if len(gold) == len(pred): - if isinstance(gold, FiniteSet): - gold_args = list(ordered(gold.args, keys=sort_key, default=False)) - pred_args = list(ordered(pred.args, keys=sort_key, default=False)) - - elif isinstance(gold, Tuple) and isinstance(pred, L2SFiniteSet): - # We treat the pred as tuple too - pred_args = pred._unsorted_args - gold_args = gold.args - - elif isinstance(pred, FiniteSet): - pred_args = list(ordered(pred.args, keys=sort_key, default=False)) - gold_args = gold.args - else: - gold_args = gold.args - pred_args = pred.args - - return all(sympy_expr_eq(a, b, precision) for a, b in zip(gold_args, pred_args)) - - return False - - -def sympy_compare_symbols(gold: Basic | MatrixBase, pred: Basic | MatrixBase) -> bool: - """Compare two sympy expressions where at least one is a Symbol. - - Handles special cases: - - One is Symbol and other is E (limitation of parsed expressions) - - One is multiplication of symbols and other is single symbol (concatenated comparison) - """ - # Handle E vs symbol case - if (isinstance(gold, Symbol) and gold.name.lower() == "e" and pred == E) or ( - isinstance(pred, Symbol) and pred.name.lower() == "e" and gold == E - ): - return True - - # Handle multiplication of symbols vs single symbol - if ( - isinstance(gold, Symbol) - and isinstance(pred, Mul) - and all(arg == E or isinstance(arg, (Symbol)) for arg in pred.args) - ): - concat_pred = "".join(arg.name if isinstance(arg, Symbol) else "e" for arg in pred.args) - return gold.name.lower() == concat_pred.lower() - - if ( - isinstance(pred, Symbol) - and isinstance(gold, Mul) - and all(arg == E or isinstance(arg, (Symbol)) for arg in gold.args) - ): - concat_gold = "".join(arg.name if isinstance(arg, Symbol) else "e" for arg in gold.args) - return pred.name.lower() == concat_gold.lower() - - return gold == pred - - -def is_relation(expr: Basic | MatrixBase) -> bool: - """Check if an expression is a relational expression.""" - if isinstance(expr, Relational): - return True - - if isinstance(expr, And): - return all(isinstance(arg, Relational) for arg in expr.args) - - return False - - -def take_last_relation(expr: And | Relational) -> Relational: - """Take the last relation from an And expression.""" - if isinstance(expr, And): - return take_last_relation(expr.args[-1]) - return expr - - -def unwrap_fcs(expr: Basic | MatrixBase) -> Basic | MatrixBase: - """Unwrap function calls to their arguments.""" - if not isinstance(expr, Basic): - return expr - - if hasattr(expr, "func") and isinstance(expr.func, UndefinedFunction): - func_name = expr.func.__name__ - unwrapped_args = [str(unwrap_fcs(arg)) for arg in expr.args] - return Symbol(f"{func_name}_{'_'.join(unwrapped_args)}") - - try: - new_args = [unwrap_fcs(arg) for arg in expr.args] - if new_args: - return expr.func(*new_args) - except TimeoutError: - raise - except Exception: # noqa: E722 - pass - - return expr - - -def is_equation(expr: Basic | MatrixBase) -> bool: - """Check if an expression is an equation. - - Args: - expr: The expression to check - Returns: - True if expr is an equation, False otherwise - """ - if isinstance(expr, Eq): - return True - - if isinstance(expr, And) and len(expr.args) > 0: - return all(isinstance(arg, Eq) for arg in expr.args) - - return False - - -@requires("latex2sympy2_extended") -def is_assignment_relation(expr: Basic | MatrixBase) -> bool: - from latex2sympy2_extended.latex2sympy2 import is_expr_of_only_symbols - - """Check if an expression is an assignment relation. E.g a=1 - - Args: - expr: The expression to check - Returns: - bool: True if expr is a relational expression or And of relations, False otherwise - """ - if isinstance(expr, Eq) and is_expr_of_only_symbols(expr.lhs): - return True - - if isinstance(expr, And) and len(expr.args) > 0: - return all(isinstance(arg, Eq) for arg in expr.args) and is_expr_of_only_symbols(expr.args[0].lhs) - - return False - - -def sympy_compare_interval(a: Interval, b: Interval, precision: int) -> bool: - """Compare two intervals. - - Args: - a: First interval - b: Second interval - precision: Number of decimal places to compare endpoints - - Returns: - True if intervals are equal, False otherwise - """ - return ( - a.left_open == b.left_open - and a.right_open == b.right_open - and sympy_expr_eq(a.start, b.start, precision) - and sympy_expr_eq(a.end, b.end, precision) - ) - - -def sympy_compare_relational(gold: Relational | And, pred: Relational | And, precision: int) -> bool: - """Compare two relational expressions. - - Args: - gold: First relational expression - pred: Second relational expression - precision: Number of decimal places to compare - - Returns: - True if relations are equivalent, False otherwise - """ - # Handle And expressions by comparing each relation - if isinstance(gold, And): - return all(sympy_compare_relational(g, p, precision) for g, p in zip(gold.args, pred.args)) - - # Helper to check if expressions are equivalent when flipped - def are_flipped_inequalities_equal(a: Relational, b: Relational) -> bool: - try: - return sympy_expr_eq(a.lhs - a.rhs, b.rhs - b.lhs, precision) # type: ignore - except TimeoutError: - raise - except Exception: # noqa: E722 - pass - return False - - # Same type of relation (e.g. both <= or both >=) - try: - if type(gold) is type(pred) and sympy_expr_eq(gold.lhs - gold.rhs, pred.lhs - pred.rhs, precision): # type: ignore - return True - except TimeoutError: - raise - except Exception: # noqa: E722 - pass - - # Check flipped inequalities (a <= b equals b >= a) - if ( - isinstance(gold, GreaterThan) - and isinstance(pred, LessThan) - or isinstance(gold, LessThan) - and isinstance(pred, GreaterThan) - or isinstance(gold, StrictGreaterThan) - and isinstance(pred, StrictLessThan) - or isinstance(gold, StrictLessThan) - and isinstance(pred, StrictGreaterThan) - or isinstance(gold, Eq) - and isinstance(pred, Eq) - ) and are_flipped_inequalities_equal(gold, pred): - return True - - return False - - -def sympy_str_eq(a: Basic | MatrixBase, b: Basic | MatrixBase) -> bool: - """Compare two sympy expressions by string representation. - - Args: - a: First sympy expression - b: Second sympy expression - - Returns: - True if string representations are equal, False otherwise - """ - a_doit = safe_sympy_doit(a) - b_doit = safe_sympy_doit(b) - - try: - # Structural equality, the cheapest but the dumbest one, it will fail for a + b vs b + a - if a_doit == b_doit: - return True - # Then do a simple str comparison - if str(a_doit).strip() == str(b_doit).strip(): - return True - except TimeoutError: - raise - except Exception: # noqa: E722 - pass - return False - - -def sympy_compare_sets( - gold: Set | Basic | MatrixBase | Tuple, pred: Set | Basic | MatrixBase | Tuple, precision: int -) -> bool: - """Compare two sympy sets for equality using multiple methods. - - Args: - gold: First sympy set (expected) - pred: Second sympy set (predicted) - precision: Number of decimal places to compare - - Returns: - True if sets are equal by any comparison method, False otherwise - """ - # Convert non-sets to singleton sets - a_set = gold if isinstance(gold, (Set, Tuple)) else FiniteSet(gold) - b_set = pred if isinstance(pred, (Set, Tuple)) else FiniteSet(pred) - - # If both are intervals, use interval comparison - if isinstance(a_set, Interval) and isinstance(b_set, Interval): - return sympy_compare_interval(a_set, b_set, precision) - - # Try direct set equality - if a_set == b_set: - return True - - # If both are sets, check if they are equal - if isinstance(a_set, Set) and isinstance(b_set, Set) and a_set.symmetric_difference(b_set).is_empty: - return True - - # For finite sets, compare elements - if isinstance(a_set, (FiniteSet, Tuple)) and isinstance(b_set, (FiniteSet, Tuple)): - return sympy_deep_compare_set_and_tuple(a_set, b_set, precision) - - # Because (1,2) is parsed as Interval(1,2,left_open=True,right_open=True), it could have that the - # correct is (1,2) and predicted is 1,2, which is parsed as Set(1,2) - if isinstance(a_set, Interval) and isinstance(b_set, (FiniteSet, Tuple)): - if a_set.is_open and len(b_set) == 2: - return sympy_deep_compare_set_and_tuple(Tuple(a_set.start, a_set.end), b_set, precision) - - if isinstance(b_set, Interval) and isinstance(a_set, (FiniteSet, Tuple)): - if b_set.is_open and len(a_set) == 2: - return sympy_deep_compare_set_and_tuple(a_set, Tuple(b_set.start, b_set.end), precision) - - return False - - -def sympy_expr_eq(gold: Basic | MatrixBase, pred: Basic | MatrixBase, precision: int, strict: bool = True) -> bool: # noqa: C901 - """Compare two sympy expressions for equality using multiple methods. - - Args: - gold: First sympy expression (expected) - pred: Second sympy expression (predicted) - precision: Number of decimal places to compare - strict: If true, variables do matter otherwise they don't - - Returns: - True if expressions are equal by any comparison method, False otherwise - """ - # This ensures that f(x) == f(y) is true - if not strict: - try: - gold_variables = gold.free_symbols - pred_variables = pred.free_symbols - if len(gold_variables) == len(pred_variables): - pred = pred.subs(list(zip(pred_variables, gold_variables))) - except TimeoutError: - raise - except Exception: # noqa: E722 - pass - - # If the target is relational, but the refernce is not, it's possible it's a case of a=x+1+z, so we just take x+1+z - # We only do this if the lhs of the first equation is fully symbolic, to prevent simplifying x+y+2z = 1 - if is_assignment_relation(gold) and not is_equation(pred): - gold = take_last_relation(gold).rhs - - # Here we respect the gold and simplify accordingly, thus any of - # k=x+1+z or 1+1+1=3 will be simplified to rhs - if is_equation(pred) and not is_equation(gold): - pred = take_last_relation(pred).rhs - - if is_relation(gold) and isinstance(pred, Set): - # This is to ensure that 1 < x < 2 equals (-oo, 1) U (2, oo) - # We also unwrap the functions because otherwise it creates some conditional set based on the function name - try: - gold = unwrap_fcs(gold).as_set() - except TimeoutError: - raise - except Exception: # noqa: E722 - pass - - # Start with simple str and expr comparison as it's the fastest - # str comparison is better than simple eq, because it will also handle misarrangements - if sympy_str_eq(gold, pred): - return True - - # Support for equations - if is_relation(gold) and is_relation(pred): - return sympy_compare_relational(gold, pred, precision) - - elif isinstance(gold, (Set, Tuple)) or isinstance(pred, (Set, Tuple)): - return sympy_compare_sets(gold, pred, precision) - - # Handles $\text{answer}$ == $answer$, one is symbol, is multiplication of symbols (a*n*s*w*e*r) - elif isinstance(gold, Symbol) or isinstance(pred, Symbol): - return sympy_compare_symbols(gold, pred) - - elif isinstance(gold, (Basic, MatrixBase)) and isinstance(pred, (Basic, MatrixBase)): - # Mostly so that 0.333333 = 1/3 - if sympy_numeric_eq(gold, pred, precision): - return True - # Then try symbolic equality - if sympy_symbolic_eq(gold, pred): - return True - - return False - - -complex_number_pattern = re.compile( - r""" - # Complex number indicators - \\mathbb\{C\}| # Complex number set ℂ - \\i\b| # Complex i - \bi\b| # Standalone i - \\text\{i\}| # Text i - \\mathrm\{i\}| # Roman i - \\imath\b| # Alternative i notation - - # Matrix operations - \\det| # Determinant - \\operatorname\{tr\}| # Trace - \\operatorname\{rank\}| # Rank - \\text\{rank\}| - \\arg\{| # Complex argument - \\Re\{| # Real part - \\Im\{| # Imaginary part - \\operatorname\{Re\}| # Real part alternate - \\operatorname\{Im\}| # Imaginary part alternate - \\text\{Re\}| # Real part text - \\text\{Im\} # Imaginary part text -""", - re.VERBOSE, -) - - -def should_treat_as_complex(latex_str: str) -> bool: - """Returns True if the latex string likely contains complex numbers, matrices, or vectors.""" - return bool(complex_number_pattern.search(latex_str)) - - -def compare_gold_target( - gold: list[Basic | MatrixBase | str] | Basic | MatrixBase | str, - target: list[Basic | MatrixBase | str] | Basic | MatrixBase | str, - precision: int = 6, - strict: bool = True, - timeout_seconds: int = 3, -) -> bool: - @timeout(timeout_seconds=timeout_seconds) - def compare_single_extraction(gold: Basic | MatrixBase | str, target: Basic | MatrixBase | str) -> bool: - # If both are sympy expressions, we can use sympy to compare them - if isinstance(gold, (Basic, MatrixBase)) and isinstance(target, (Basic, MatrixBase)): - return sympy_expr_eq(gold, target, precision, strict) - - # We don't support str / sympy.Expr comparison. Imo there is no point in doing this, as chances - # of this happening are very low. The only reason why one of them is not converted to sympy expression - # is usually because the parsing logic failed in this case we should improve the parsing logic - # instead of somehow fixing adhoc. - elif isinstance(gold, str) and isinstance(target, str): - # We just do string comparison for everything else - gold = gold.strip() - target = target.strip() - - # Ensure it's both not empty and equal - return len(gold) > 0 and len(target) > 0 and gold == target - - return False - - def compare_single_extraction_wrapper(g, t): - try: - return compare_single_extraction(g, t) - except Exception: # noqa: E722 - return False - - return any(compare_single_extraction_wrapper(g, t) for g, t in product(gold, target)) diff --git a/src/lighteval/metrics/utils/metric_utils.py b/src/lighteval/metrics/utils/metric_utils.py deleted file mode 100644 index adebda5e5..000000000 --- a/src/lighteval/metrics/utils/metric_utils.py +++ /dev/null @@ -1,141 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from dataclasses import dataclass -from typing import Callable - -from lighteval.metrics.metrics_corpus import CorpusLevelComputation -from lighteval.metrics.metrics_sample import SampleLevelComputation -from lighteval.metrics.sample_preparator import Preparator -from lighteval.tasks.requests import SamplingMethod - - -@dataclass -class Metric: - metric_name: str - higher_is_better: bool - category: SamplingMethod - sample_level_fn: SampleLevelComputation | Preparator - corpus_level_fn: CorpusLevelComputation | Callable - - batched_compute: bool = False - - def get_doc(self): - return self.sample_level_fn.__doc__ - - def compute_sample( - self, **kwargs - ) -> dict: # result: Union[list[ModelResponse], ModelResponse], formatted_doc: Doc) -> dict: - if isinstance(self.sample_level_fn, SampleLevelComputation): - sample_level_fn = self.sample_level_fn.compute - elif isinstance(self.sample_level_fn, Preparator): - sample_level_fn = self.sample_level_fn.prepare - else: - raise ValueError( - f"Incorrect type for {self.sample_level_fn}, should be a SampleLevelComputation or Preparator" - ) - - if isinstance(self, MetricGrouping): - return sample_level_fn(**kwargs) - return {self.metric_name: sample_level_fn(**kwargs)} - - def get_corpus_aggregations(self) -> dict: - if isinstance(self, MetricGrouping): - if isinstance(self.corpus_level_fn, dict): - corpus_level_fn = self.corpus_level_fn - else: - corpus_level_fn = dict.fromkeys(self.metric_name, self.corpus_level_fn) - else: - corpus_level_fn = {self.metric_name: self.corpus_level_fn} - - for name, item in corpus_level_fn.items(): - if isinstance(item, Callable): - corpus_level_fn[name] = item - else: - corpus_level_fn[name] = item.compute_corpus - - return corpus_level_fn - - def __call__(self, sample_params: dict | None): - """Allow creating new instances with modified parameters""" - if sample_params is not None: - for k, v in sample_params.items(): - setattr(self.sample_level_fn, k, v) - - # Once the parameters are updated, we need to adjust the - # metric name to what will be returned - # CAREFUL: do not change the following logic! - # It must always provide the values of all parameters, so that people can evaluate using a range of metrics - # For example, pass@k=1&n=16, pass@k=10&n=16, etc - sample_params_name = "&".join(f"{k}={v}" for k, v in sample_params.items()) - if isinstance(self, MetricGrouping): - if hasattr(self.sample_level_fn, "metric_names"): - # this is mostly for the gpass@k metrics which redefine submetric names - self.metric_name = self.sample_level_fn.metric_names - else: - self.metric_name = [f"{metric}:{sample_params_name}" for metric in self.metric_name] - else: - self.metric_name = f"{self.metric_name}:{sample_params_name}" - return self - - @staticmethod - def get_allowed_types_for_metrics(): - return (SampleLevelComputation, Preparator, CorpusLevelComputation, Callable) - - -@dataclass -class MetricGrouping(Metric): - """Some metrics are more advantageous to compute together at once. - For example, if a costly preprocessing is the same for all metrics, it makes more sense to compute it once. - """ - - metric_name: list[str] - corpus_level_fn: dict[str, Callable] - higher_is_better: dict[str, Callable] - - -@dataclass -class CorpusLevelMetric(Metric): - """Metric computed over the whole corpora, with computations happening at the aggregation phase""" - - pass - - -@dataclass -class SampleLevelMetric(Metric): - """Metric computed per sample, then aggregated over the corpus""" - - pass - - -@dataclass -class CorpusLevelMetricGrouping(MetricGrouping): - """MetricGrouping computed over the whole corpora, with computations happening at the aggregation phase""" - - pass - - -@dataclass -class SampleLevelMetricGrouping(MetricGrouping): - """MetricGrouping are computed per sample, then aggregated over the corpus""" - - pass diff --git a/src/lighteval/metrics/utils/stderr.py b/src/lighteval/metrics/utils/stderr.py deleted file mode 100644 index e8d47e444..000000000 --- a/src/lighteval/metrics/utils/stderr.py +++ /dev/null @@ -1,123 +0,0 @@ -# MIT License - -# Copyright (c) 2024 Eleuther AI and The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -# This section of the code comes almost entirely from the harness -# We kept it because it's very fast - however, we renamed the variables -# and added documentation - -import logging -import math -import os -import random -from typing import Callable, Optional - -import numpy as np -from scipy.stats import bootstrap -from tqdm import tqdm - - -logger = logging.getLogger(__name__) - -LIGHTEVAL_DISABLE_TQDM = int(os.environ.get("LIGHTEVAL_DISABLE_TQDM", "0")) - - -def _stddev(arr): - mu = np.mean(arr) - return math.sqrt(sum((x - mu) ** 2 for x in arr) / (len(arr) - 1)) - - -def mean_stderr(arr): - return _stddev(arr) / math.sqrt(len(arr)) - - -class _bootstrap_internal: - def __init__(self, number_draws: int, metric: Optional[Callable] = None): - self.number_draws = number_draws - self.metric = metric - - def __call__(self, cur_experiment): - # Creates number_draws samplings (with replacement) of the population by iterating on a given seed - population, seed = cur_experiment - rnd = random.Random() - rnd.seed(seed) - samplings = [] - import multiprocessing as mp - - with mp.Pool(mp.cpu_count()) as pool: - samplings = pool.starmap( - self.metric, - tqdm( - [(rnd.choices(population, k=len(population)),) for _ in range(self.number_draws)], - total=self.number_draws, - desc="Sampling bootstrap iterations", - disable=bool(LIGHTEVAL_DISABLE_TQDM), - ), - ) - return samplings - - -def bootstrap_stderr(metric: Callable, population: list, number_experiments: int): - """Bootstraps the stderr of the given metric for the given population of samples, - by sampling said population for number_experiments and recomputing the metric on the - different samplings. - """ - res = [] - number_draws = min(1000, number_experiments) - number_seeds = number_experiments // number_draws - - logger.info(f"Bootstrapping {metric.__name__}'s stderr with {number_seeds} seeds.") - for seed in range(number_seeds): - # sample w replacement - res.extend(_bootstrap_internal(metric=metric, number_draws=number_draws)((population, seed))) - - return mean_stderr(res) - - -def get_stderr_function(aggregation: Callable, number_experiments: int = 1000): - # Mean stderr can be computed trivially - if "mean" in aggregation.__name__: - return mean_stderr - - # For other metrics, we bootstrap the stderr by sampling - try: - return lambda population: bootstrap_stderr( - metric=aggregation, population=population, number_experiments=number_experiments - ) - except Exception: - return None - - -def bootstrap_stderr_scipy(metric: Callable, population: list, number_experiments: int = 1000): - """Simulates resampling (draw with replacement, of the same size as the orig set) n times from the results population - to compute the distance between the simulated resampling distribution and the actual distribution. - Same as bootstrap_stderr, but uses scipy. - It's kept for archive, as it overflows for big datasets - """ - logger.info(f"Bootstrapping {metric.__name__}'s stderr.") - res = bootstrap( - data=[population], - statistic=metric, - n_resamples=number_experiments, - confidence_level=0.95, - method="BCa", - ) - return res.standard_error diff --git a/src/lighteval/models/abstract_model.py b/src/lighteval/models/abstract_model.py deleted file mode 100644 index d9d5b4100..000000000 --- a/src/lighteval/models/abstract_model.py +++ /dev/null @@ -1,351 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import json -import re -from abc import ABC, abstractmethod -from typing import Optional, Union - -import torch -import yaml -from pydantic import BaseModel -from transformers.tokenization_utils_base import BatchEncoding, PreTrainedTokenizerBase - -from lighteval.models.model_input import GenerationParameters -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.requests import Doc - - -TokenSequence = Union[list[int], torch.LongTensor, torch.Tensor, BatchEncoding] - - -class ModelConfig(BaseModel, extra="forbid"): - """Base configuration class for all model types in Lighteval. - - This is the foundation class that all specific model configurations inherit from. - It provides common functionality for parsing configuration from files and command-line arguments, - as well as shared attributes that are used by all models like generation parameters and system prompts. - - Attributes: - model_name (str): - The model name or unique id - generation_parameters (GenerationParameters): - Configuration parameters that control text generation behavior, including - temperature, top_p, max_new_tokens, etc. Defaults to empty GenerationParameters. - system_prompt (str | None): - Optional system prompt to be used with chat models. This prompt sets the - behavior and context for the model during evaluation. - cache_dir (str): - Directory to cache the model. Defaults to "~/.cache/huggingface/lighteval". - - Methods: - from_path(path: str): - Load configuration from a YAML file. - from_args(args: str): - Parse configuration from a command-line argument string. - _parse_args(args: str): - Static method to parse argument strings into configuration dictionaries. - - Example: - ```python - # Load from YAML file - config = ModelConfig.from_path("model_config.yaml") - - # Load from command line arguments - config = ModelConfig.from_args("model_name=meta-llama/Llama-3.1-8B-Instruct,system_prompt='You are a helpful assistant.',generation_parameters={temperature=0.7}") - - # Direct instantiation - config = ModelConfig( - model_name="meta-llama/Llama-3.1-8B-Instruct", - generation_parameters=GenerationParameters(temperature=0.7), - system_prompt="You are a helpful assistant." - ) - ``` - """ - - model_name: str = None - - generation_parameters: GenerationParameters = GenerationParameters() - system_prompt: str | None = None - cache_dir: str = "~/.cache/huggingface/lighteval" - - @classmethod - def from_path(cls, path: str): - with open(path, "r") as f: - config = yaml.safe_load(f) - - return cls(**config["model_parameters"]) - - @classmethod - def from_args(cls, args: str): - config = cls._parse_args(args) - return cls(**config) - - @staticmethod - def _parse_args(args: str) -> dict: - """Parse a string of arguments into a configuration dictionary. - - This function parses a string containing model arguments and generation parameters - into a structured dictionary with two main sections: 'model' and 'generation'. - It specifically handles generation parameters enclosed in curly braces. - - Args: - args (str): A string containing comma-separated key-value pairs, where generation - parameters can be specified in a nested JSON-like format. - - Returns: - dict: A dictionary with two keys: - - 'model': Contains general model configuration parameters - - 'generation': Contains generation-specific parameters - - Examples: - >>> parse_args("model_name=gpt2,max_length=100") - { - 'model': {'model_name': 'gpt2', 'max_length': '100'}, - } - - >>> parse_args("model_name=gpt2,generation_parameters={temperature:0.7,top_p:0.9}") - { - 'model': {'model_name': 'gpt2', 'generation_parameters': {'temperature': 0.7, 'top_p': 0.9}, - } - - >>> parse_args("model_name=gpt2,use_cache,generation_parameters={temperature:0.7}") - { - 'model': {'model_name': 'gpt2', 'use_cache': True, 'generation_parameters': {'temperature': 0.7}}, - } - """ - # Looking for generation_parameters in the model_args - generation_parameters_dict = None - pattern = re.compile(r"(\w+)=(\{.*\}|[^,]+)") - matches = pattern.findall(args) - for key, value in matches: - key = key.strip() - if key == "generation_parameters": - # Keys must be quoted (since they are strings) - gen_params = re.sub(r"(\w+):", r'"\1":', value) - # for k, v where v are strings, we quote them too - gen_params = re.sub(r":\s*([A-Za-z_][\w.-]*)\s*(?=[,}])", r':"\1"', gen_params) - generation_parameters_dict = json.loads(gen_params) - - args = re.sub(r"generation_parameters=\{.*?\},?", "", args).strip(",") - model_config = {k.split("=")[0]: k.split("=")[1] if "=" in k else True for k in args.split(",")} - - if generation_parameters_dict is not None: - model_config["generation_parameters"] = generation_parameters_dict - - return model_config - - -class InspectAIModelConfig(BaseModel): - max_tokens: int | None = None - system_message: str | None = None - temperature: float | None = None - top_p: float | None = None - top_k: int | None = None - frequence_penalty: float | None = None - presence_penalty: float | None = None - seed: int | None = None - stop_seqs: list[str] | None = None - num_choices: int | None = None - best_of: int | None = None - log_probs: bool | None = None - top_logprobs: int | None = None - cache_prompt: bool | None = None - reasoning_effort: int | None = None - reasoning_tokens: int | None = None - reasoning_history: bool | None = None - response_format: str | None = None - parallel_tool_calls: bool | None = None - max_tool_output: int | None = None - internal_tools: bool | None = None - - @staticmethod - def _parse_args(args: str) -> dict: - """Parse a string of arguments into a configuration dictionary. - - This function parses a string containing model arguments and generation parameters - into a structured dictionary with two main sections: 'model' and 'generation'. - It specifically handles generation parameters enclosed in curly braces. - - Args: - args (str): A string containing comma-separated key-value pairs, where generation - parameters can be specified in a nested JSON-like format. - - Returns: - dict: A dictionary with two keys: - - 'model': Contains general model configuration parameters - - 'generation': Contains generation-specific parameters - - Examples: - >>> parse_args("max_length=100") - { - 'max_tokens': '100', - } - """ - args = re.sub(r"generation_parameters=\{.*?\},?", "", args).strip(",") - return {k.split("=")[0]: k.split("=")[1] if "=" in k else True for k in args.split(",")} - - @classmethod - def from_path(cls, path: str): - with open(path, "r") as f: - config = yaml.safe_load(f) - - return cls(**config["model_parameters"]) - - @classmethod - def from_args(cls, args: str): - config = cls._parse_args(args) - return cls(**config) - - -class LightevalModel(ABC): - DATASET_SPLITS = 4 - is_async = False - - """Abstract model class defining the API that every model to plug into lighteval must follow.""" - - def cleanup(self): - """Clean up operations if needed, such as closing an endpoint.""" - return - - @property - @abstractmethod - def tokenizer(self) -> PreTrainedTokenizerBase: - raise NotImplementedError - - @property - @abstractmethod - def add_special_tokens(self) -> bool: - raise NotImplementedError - - @property - @abstractmethod - def max_length(self) -> int: - """Return the maximum sequence length of the model.""" - raise NotImplementedError - - @property - def disable_tqdm(self) -> bool: - return False - - @abstractmethod - def greedy_until( - self, - docs: list[Doc], - ) -> list[ModelResponse]: - """Generates responses using a greedy decoding strategy until certain ending conditions are met. - - Args: - docs (list[Doc]): List of documents containing the context for generation. - - Returns: - list[ModelResponse]: list of generated responses. - """ - return NotImplemented - - @abstractmethod - def loglikelihood(self, docs: list[Doc]) -> list[ModelResponse]: - """Tokenize the context and continuation and compute the log likelihood of those - tokenized sequences. - - Returns: - list[ModelResponse]: List of model responses containing log likelihood scores - """ - return NotImplemented - - @abstractmethod - def loglikelihood_rolling(self, docs: list[Doc]) -> list[ModelResponse]: - """This function is used to compute the log likelihood of the context for perplexity metrics. - - Returns: - list[ModelResponse]: List of model responses containing log likelihood scores - """ - return NotImplemented - - # Tokenization utils - def tok_encode(self, str_to_encode: str | list[str], add_special_tokens: Optional[bool] = None) -> TokenSequence: - if add_special_tokens is None: - add_special_tokens = self.add_special_tokens - if isinstance(str_to_encode, str): - return self.tokenizer.encode(str_to_encode, add_special_tokens=add_special_tokens) - return self.tokenizer( - str_to_encode, - padding=True, - add_special_tokens=add_special_tokens, - return_tensors="pt", - ) - - def tok_encode_pair(self, context, continuations: list[str], pairwise: bool = False): - """Encodes a context with a list of continuations by taking care of the spaces in between. - - Args: - context (str): The context string to be encoded. - continuations (list[str]): List of continuation strings to be encoded. - pairwise (bool): - If True, encode context and continuations separately. - If False, encode them together and then split. - - Returns: - Tuple[TokenSequence, list[TokenSequence]]: - A tuple containing the encoded context and a list of encoded continuations. - - The advantage of pairwise is: - 1) It better aligns with how LLM predicts tokens - 2) Works in case len(tok(context,cont)) != len(tok(context)) + len(tok(continuation)). - E.g this can happen for chinese if no space is used between context/continuation - """ - n_spaces = len(context) - len(context.rstrip()) - if n_spaces > 0: - continuations = [context[-n_spaces:] + cont for cont in continuations] - context = context[:-n_spaces] - - if pairwise: - # We don't add special tokens to the continuation as if bos is added - # models tend to to completely ignore a context - context_enc = self.tok_encode(context, add_special_tokens=self.add_special_tokens) - continuation_enc = [self.tok_encode(cont, add_special_tokens=False) for cont in continuations] - - # In theory the context_enc can be ended with eos token, this would again - # cause the model to ignore the context. We thus strip the eos token from context_enc - if len(context_enc) > 0 and context_enc[-1] == self.tokenizer.eos_token_id: - context_enc = context_enc[:-1] - - context_encs = [context_enc] * len(continuation_enc) - - return context_encs, continuation_enc - - # Handle list of continuations - context_enc = self.tok_encode(context) - context_encs = [] - continuations_encs = [] - for cont in continuations: - whole_enc = self.tok_encode(context + cont) - context_enc_len = len(context_enc) - if len(context_enc) == len(whole_enc): - context_enc_len = len(context_enc) - 1 - continuations_encs.append(whole_enc[context_enc_len:]) - context_encs.append(whole_enc[:context_enc_len]) - - return context_encs, continuations_encs - - def tok_decode(self, tokens: torch.LongTensor) -> list[str]: - return self.tokenizer.batch_decode(tokens, skip_special_tokens=True) diff --git a/src/lighteval/models/custom/custom_model.py b/src/lighteval/models/custom/custom_model.py deleted file mode 100644 index ea11fe3c6..000000000 --- a/src/lighteval/models/custom/custom_model.py +++ /dev/null @@ -1,73 +0,0 @@ -# MIT License -# -# Copyright (c) 2024 The HuggingFace Team -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from lighteval.models.abstract_model import ModelConfig - - -class CustomModelConfig(ModelConfig): - """Configuration class for loading custom model implementations in Lighteval. - - This config allows users to define and load their own model implementations by specifying - a Python file containing a custom model class that inherits from LightevalModel. - - The custom model file should contain exactly one class that inherits from LightevalModel. - This class will be automatically detected and instantiated when loading the model. - - Args: - model (str): - An identifier for the model. This can be used to track which model was evaluated - in the results and logs. - - model_definition_file_path (str): - Path to a Python file containing the custom model implementation. This file must - define exactly one class that inherits from LightevalModel. The class should - implement all required methods from the LightevalModel interface. - - Example usage: - ```python - # Define config - config = CustomModelConfig( - model="my-custom-model", - model_definition_file_path="path/to/my_model.py" - ) - - # Example custom model file (my_model.py): - from lighteval.models.abstract_model import LightevalModel - - class MyCustomModel(LightevalModel): - def __init__(self, config, env_config): - super().__init__(config, env_config) - # Custom initialization... - - def greedy_until(self, docs: list[Doc]) -> list[ModelResponse]: - # Custom generation logic... - pass - - def loglikelihood(self, docs: list[Doc]) -> list[ModelResponse]: - pass - ``` - - An example of a custom model can be found in `examples/custom_models/google_translate_model.py`. - """ - - model_name: str - model_definition_file_path: str diff --git a/src/lighteval/models/dummy/dummy_model.py b/src/lighteval/models/dummy/dummy_model.py deleted file mode 100644 index e0a13b589..000000000 --- a/src/lighteval/models/dummy/dummy_model.py +++ /dev/null @@ -1,118 +0,0 @@ -# MIT License -# -# Copyright (c) 2024 The HuggingFace Team -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -# inspired by https://github.com/EleutherAI/lm-evaluation-harness/blob/main/lm_eval/models/dummy.py - -import random - -from transformers.models.auto.tokenization_auto import AutoTokenizer - -from lighteval.models.abstract_model import LightevalModel, ModelConfig -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.requests import Doc, SamplingMethod -from lighteval.utils.cache_management import SampleCache, cached - - -class DummyModelConfig(ModelConfig): - """Configuration class for dummy models used for testing and baselines. - - This configuration is used to create dummy models that generate random responses - or baselines for evaluation purposes. Useful for testing evaluation pipelines - without requiring actual model inference. - - Attributes: - model_name (str): - Name of your choice - "dummy" by default - seed (int): - Random seed for reproducible dummy responses. Defaults to 42. - This seed controls the randomness of the generated responses and log probabilities. - - Example: - ```python - config = DummyModelConfig( - model_name="my_dummy", - seed=123, - ) - ``` - """ - - model_name: str = "dummy" - seed: int = 42 - - -class DummyModel(LightevalModel): - """Dummy model to generate random baselines.""" - - def __init__( - self, - config: DummyModelConfig, - ): - self.config = config - self._random = random.Random(self.config.seed) - self._tokenizer = None - - # Initialize cache for tokenization and predictions - self._cache = SampleCache(config) - - @property - def tokenizer(self): - if not self._tokenizer: - self._tokenizer = AutoTokenizer.from_pretrained("gpt2") - return self._tokenizer - - @property - def add_special_tokens(self): - return False - - @property - def max_length(self) -> int: - return 2048 - - @cached(SamplingMethod.GENERATIVE) - def greedy_until(self, docs: list[Doc]) -> list[ModelResponse]: - return [ModelResponse(text=["random baseline"]) for _ in range(len(docs))] - - @cached(SamplingMethod.LOGPROBS) - def loglikelihood(self, docs: list[Doc]) -> list[ModelResponse]: - model_responses = [] - for doc in docs: - model_responses.append( - ModelResponse( - logprobs=[-self._random.random() for _ in doc.choices], - argmax_logits_eq_gold=[False for _ in doc.choices], - ) - ) - - return model_responses - - @cached(SamplingMethod.PERPLEXITY) - def loglikelihood_rolling(self, docs: list[Doc]) -> list[ModelResponse]: - model_responses = [] - for doc in docs: - model_responses.append( - ModelResponse( - logprobs=[-self._random.random() for _ in doc.choices], - argmax_logits_eq_gold=[False for _ in doc.choices], - ) - ) - - return model_responses diff --git a/src/lighteval/models/endpoints/endpoint_model.py b/src/lighteval/models/endpoints/endpoint_model.py deleted file mode 100644 index 6b08be575..000000000 --- a/src/lighteval/models/endpoints/endpoint_model.py +++ /dev/null @@ -1,674 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import asyncio -import logging -import re -import time -from typing import Coroutine, Dict, List, Optional, Tuple, Union - -import requests -import torch -from huggingface_hub import ( - AsyncInferenceClient, - InferenceClient, - InferenceEndpoint, - InferenceEndpointError, - TextGenerationInputGrammarType, - TextGenerationOutput, - create_inference_endpoint, - get_inference_endpoint, -) -from huggingface_hub.errors import HfHubHTTPError -from requests import ConnectionError -from torch.utils.data import DataLoader -from tqdm import tqdm -from transformers.models.auto.tokenization_auto import AutoTokenizer - -from lighteval.data import GenerativeTaskDataset, LoglikelihoodDataset -from lighteval.models.abstract_model import LightevalModel, ModelConfig -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.prompt_manager import PromptManager -from lighteval.tasks.requests import Doc, SamplingMethod -from lighteval.utils.cache_management import SampleCache, cached - - -logger = logging.getLogger(__name__) - -BATCH_SIZE = 50 -MAX_TIME_FOR_SPINUP = 3600 - -SORTED_INSTANCE_SIZES = [ # sorted by incremental overall RAM (to load models) - # type, size - ("nvidia-a10g", "x1"), - ("nvidia-t4", "x4"), - ("nvidia-a100", "x1"), - ("nvidia-a10g", "x4"), - ("nvidia-a100", "x2"), - ("nvidia-a100", "x4"), -] - - -class ServerlessEndpointModelConfig(ModelConfig): - """Configuration class for HuggingFace Inference API (inference endpoints). - - https://huggingface.co/inference-endpoints/dedicated - - Attributes: - model_name (str): - HuggingFace Hub model ID to use with the Inference API. - Example: "meta-llama/Llama-3.1-8B-Instruct" - add_special_tokens (bool): - Whether to add special tokens during tokenization. Defaults to True. - batch_size (int): - Batch size for requests. Defaults to 1 (serverless API limitation). - generation_parameters (GenerationParameters, optional, defaults to empty GenerationParameters): - Configuration parameters that control text generation behavior, including - temperature, top_p, max_new_tokens, etc. - system_prompt (str | None, optional, defaults to None): Optional system prompt to be used with chat models. - This prompt sets the behavior and context for the model during evaluation. - cache_dir (str, optional, defaults to "~/.cache/huggingface/lighteval"): Directory to cache the model. - - Example: - ```python - config = ServerlessEndpointModelConfig( - model_name="meta-llama/Llama-3.1-8B-Instruct", - generation_parameters=GenerationParameters( - temperature=0.7, - max_new_tokens=100 - ) - ) - ``` - """ - - model_name: str - add_special_tokens: bool = True - batch_size: int = 1 - - -class InferenceEndpointModelConfig(ModelConfig): - """Configuration class for HuggingFace Inference Endpoints (dedicated infrastructure). - - This configuration is used to create and manage dedicated inference endpoints - on HuggingFace's infrastructure. These endpoints provide dedicated compute - resources and can handle larger batch sizes and higher throughput. - - Attributes: - endpoint_name (str | None): - Name for the inference endpoint. If None, auto-generated from model_name. - model_name (str | None): - HuggingFace Hub model ID to deploy. Required if endpoint_name is None. - reuse_existing (bool): - Whether to reuse an existing endpoint with the same name. Defaults to False. - accelerator (str): - Type of accelerator to use. Defaults to "gpu". Options: "gpu", "cpu". - dtype (str | None): - Model data type. If None, uses model default. Options: "float16", "bfloat16", "awq", "gptq", "8bit", "4bit". - vendor (str): - Cloud vendor for the endpoint. Defaults to "aws". Options: "aws", "azure", "gcp". - region (str): - Cloud region for the endpoint. Defaults to "us-east-1". - instance_size (str | None): - Instance size for the endpoint. If None, auto-scaled. - instance_type (str | None): - Instance type for the endpoint. If None, auto-scaled. - framework (str): - ML framework to use. Defaults to "pytorch". - endpoint_type (str): - Type of endpoint. Defaults to "protected". Options: "protected", "public". - add_special_tokens (bool): - Whether to add special tokens during tokenization. Defaults to True. - revision (str): - Git revision of the model. Defaults to "main". - namespace (str | None): - Namespace for the endpoint. If None, uses current user's namespace. - image_url (str | None): - Custom Docker image URL. If None, uses default TGI image. - env_vars (dict | None): - Additional environment variables for the endpoint. - batch_size (int): - Batch size for requests. Defaults to 1. - generation_parameters (GenerationParameters, optional, defaults to empty GenerationParameters): - Configuration parameters that control text generation behavior, including - temperature, top_p, max_new_tokens, etc. - system_prompt (str | None, optional, defaults to None): Optional system prompt to be used with chat models. - This prompt sets the behavior and context for the model during evaluation. - cache_dir (str, optional, defaults to "~/.cache/huggingface/lighteval"): Directory to cache the model. - - Methods: - model_post_init(): - Validates configuration and ensures proper parameter combinations. - get_dtype_args(): - Returns environment variables for dtype configuration. - get_custom_env_vars(): - Returns custom environment variables for the endpoint. - - Example: - ```python - config = InferenceEndpointModelConfig( - model_name="microsoft/DialoGPT-medium", - instance_type="nvidia-a100", - instance_size="x1", - vendor="aws", - region="us-east-1", - dtype="float16", - generation_parameters=GenerationParameters( - temperature=0.7, - max_new_tokens=100 - ) - ) - ``` - - Note: - - Creates dedicated infrastructure for model inference - - Supports various quantization methods and hardware configurations - - Auto-scaling available for optimal resource utilization - - Requires HuggingFace Pro subscription for most features - - Endpoints can take several minutes to start up - - Billed based on compute usage and duration - """ - - endpoint_name: str | None = None - model_name: str | None = None - reuse_existing: bool = False - accelerator: str = "gpu" - dtype: str | None = None # if empty, we use the default - vendor: str = "aws" - region: str = "us-east-1" # this region has the most hardware options available - instance_size: str | None = None # if none, we autoscale - instance_type: str | None = None # if none, we autoscale - framework: str = "pytorch" - endpoint_type: str = "protected" - add_special_tokens: bool = True - revision: str = "main" - namespace: str | None = ( - None # The namespace under which to launch the endpoint. Defaults to the current user's namespace - ) - image_url: str | None = None - env_vars: dict | None = None - batch_size: int = 1 - - def model_post_init(self, __context): - # xor operator, one is None but not the other - if (self.instance_size is None) ^ (self.instance_type is None): - raise ValueError( - "When creating an inference endpoint, you need to specify explicitly both instance_type and instance_size, or none of them for autoscaling." - ) - - if not (self.endpoint_name is None) ^ int(self.model_name is None): - raise ValueError("You need to set either endpoint_name or model_name (but not both).") - - def get_dtype_args(self) -> Dict[str, str]: - if self.dtype is None: - return {} - model_dtype = self.dtype.lower() - if model_dtype in ["awq", "eetq", "gptq"]: - return {"QUANTIZE": model_dtype} - if model_dtype == "8bit": - return {"QUANTIZE": "bitsandbytes"} - if model_dtype == "4bit": - return {"QUANTIZE": "bitsandbytes-nf4"} - if model_dtype in ["bfloat16", "float16"]: - return {"DTYPE": model_dtype} - return {} - - def get_custom_env_vars(self) -> Dict[str, str]: - return {k: str(v) for k, v in self.env_vars.items()} if self.env_vars else {} - - -class InferenceEndpointModel(LightevalModel): - """InferenceEndpointModels can be used both with the free inference client, or with inference - endpoints, which will use text-generation-inference to deploy your model for the duration of the evaluation. - """ - - def __init__(self, config: Union[InferenceEndpointModelConfig, ServerlessEndpointModelConfig]) -> None: - self.config = config - self.reuse_existing = getattr(config, "reuse_existing", False) - self._max_length = None - self.model_name = None - self.endpoint, self.async_client, self.client = self._create_endpoint(config) - if isinstance(config, InferenceEndpointModelConfig): - self.endpoint_name = config.endpoint_name - self.name = self.endpoint.repository - self.revision = self.endpoint.revision - else: # Free inference client - self.endpoint_name = None - self.name = config.model_name - self.revision = "default" - - self.config.revision = self.revision - - self.use_async = True # set to False for debug - async use is faster - - self._tokenizer = AutoTokenizer.from_pretrained(self.name) - self._add_special_tokens = config.add_special_tokens if config.add_special_tokens is not None else False - - self.prompt_manager = PromptManager( - use_chat_template=True, tokenizer=self.tokenizer, system_prompt=config.system_prompt - ) - self.generation_parameters = config.generation_parameters - self.generation_config = self.generation_parameters.to_tgi_ie_dict() - - # Initialize cache for tokenization and predictions - self._cache = SampleCache(config) - - def _create_endpoint( # noqa: C901 - self, config: InferenceEndpointModelConfig | ServerlessEndpointModelConfig - ) -> Tuple[Union[InferenceEndpoint | None], AsyncInferenceClient, InferenceClient]: # noqa: C901 - """Creates the endpoint depending on the configuration - for an InferenceEnpointModelConfig, will retry to start - TODO: should probably split ServerlessEndpointModelConfig and make it a derived class - """ - if isinstance(config, ServerlessEndpointModelConfig): - return None, AsyncInferenceClient(model=config.model_name), InferenceClient(model=config.model_name) - - endpoint = None - - if config.instance_type and config.instance_size and config.vendor and config.region: - vendor, region, instance_type, instance_size = ( - config.vendor, - config.region, - config.instance_type, - config.instance_size, - ) - else: - try: - vendor, region, instance_type, instance_size = InferenceEndpointModel.get_suggested_model_config( - config.model_name - ) - except Exception: - vendor, region, instance_type, instance_size = ( - "aws", - "us-east-1", - *InferenceEndpointModel.get_larger_hardware_suggestion(), - ) - - must_scaleup_endpoint = False - timer_start = time.time() - # Endpoint names do not allow special characters - endpoint_name = config.endpoint_name or re.sub("[^a-zA-Z0-9-]", "-", config.model_name.lower() + "-lighteval") - # If no endpoint or endpoint not running, and we're below an hour - while (endpoint is None or endpoint.status != "running") and (time.time() - timer_start < MAX_TIME_FOR_SPINUP): - try: - if endpoint is None: # Endpoint does not exist yet locally - if not config.reuse_existing: # New endpoint - logger.info("Creating endpoint.") - endpoint: InferenceEndpoint = create_inference_endpoint( - name=endpoint_name, - namespace=config.namespace, - repository=config.model_name, - revision=config.revision, - framework=config.framework, - task="text-generation", - accelerator=config.accelerator, - type=config.endpoint_type, - vendor=vendor, - region=region, - instance_size=instance_size, - instance_type=instance_type, - custom_image={ - "health_route": "/health", - "env": { - # Documentation: https://huggingface.co/docs/text-generation-inference/en/basic_tutorials/launcher - "MAX_BATCH_PREFILL_TOKENS": "2048", - "MAX_INPUT_LENGTH": "2047", - "MAX_TOTAL_TOKENS": "2048", - "MODEL_ID": "/repository", - "HF_MODEL_TRUST_REMOTE_CODE": "true", - **config.get_dtype_args(), - **config.get_custom_env_vars(), - }, - "url": (config.image_url or "ghcr.io/huggingface/text-generation-inference:3.0.1"), - }, - ) - else: # Endpoint exists - logger.info("Reusing existing endpoint.") - endpoint = get_inference_endpoint(name=endpoint_name, namespace=config.namespace) - - else: - # Endpoint exists locally but either failed (and most likely it must be scaled up) - if must_scaleup_endpoint: - logger.info("Rescaling existing endpoint.") - endpoint.update(instance_size=instance_size, instance_type=instance_type) - must_scaleup_endpoint = False - # or we got a connection error, in which case we do nothing and just wait at the next step - - # Waits for the endpoint to be deployed - we could also check for the status in updating', 'pending', 'initializing' - logger.info("Trying to deploy your endpoint. Please wait for 10 min.") - endpoint.wait(timeout=600, refresh_every=60) # We wait for 10 min - except InferenceEndpointError as e: - logger.info( - f"Endpoint failed to start on current hardware with error {e}. Trying to autoscale to ({instance_type}, {instance_size})." - ) - instance_type, instance_size = InferenceEndpointModel.get_larger_hardware_suggestion( - instance_type, instance_size - ) - must_scaleup_endpoint = True - - except HfHubHTTPError as e: - # The endpoint actually already exists, we'll spin it up instead of trying to create a new one - if "409 Client Error: Conflict for url:" in str(e): - config.endpoint_name = endpoint_name - config.reuse_existing = True - # Requested resources are not available - elif "Bad Request: Compute instance not available yet" in str(e): - logger.error( - f"The hardware combination you are requesting does not seem to be available: ({instance_type}, {instance_size}, {config.region})." - ) - raise e - # User account does not have access to requested resources - elif "Conflict: Quota exceeded" in str(e): - raise e - except ConnectionError as e: - logger.error(f"Connection failed with error {e}. Retrying") - - if not endpoint.status == "running": - raise Exception("Did not manage to start endpoint within the elapsed time and on suggested hardware.") - - logger.info("Endpoint successfully deployed!") - - async_client: AsyncInferenceClient = endpoint.async_client - client: InferenceClient = endpoint.client - - return endpoint, async_client, client - - @staticmethod - def get_larger_hardware_suggestion(cur_instance_type: str = None, cur_instance_size: str = None): - cur_instance_ix = -1 - try: - if cur_instance_type and cur_instance_size: - cur_instance_ix = SORTED_INSTANCE_SIZES.index((cur_instance_type, cur_instance_size)) - new_instance_type = SORTED_INSTANCE_SIZES[cur_instance_ix + 1][0] - new_instance_size = SORTED_INSTANCE_SIZES[cur_instance_ix + 1][1] - return new_instance_type, new_instance_size - except ValueError: - raise Exception( - f"Problem when scaling endpoint: the current instance combination ({cur_instance_type}, {cur_instance_size}) is unknown. Can't scale it up." - ) - except IndexError: - raise Exception( - "To avoid accidental costs, we do not upgrade the current endpoint above 4 a100 automatically, please request it explicitely." - ) - - @staticmethod - def get_suggested_model_config(model_repo): - # Code from https://huggingface.co/spaces/huggingface/dedicated-endpoint-snooper/blob/main/app.py - # Example of the suggestedCompute value: 'aws-us-east-1-nvidia-l4-x1' - # -> aws us-east-1 nvidia-l4 x1 - url = f"https://ui.endpoints.huggingface.co/api/configuration?model_id={model_repo}" - response = requests.get(url) - config = response.json() - - suggested_compute = config["suggestedCompute"] - suggested_vendor = suggested_compute.split("-")[0] - if suggested_vendor == "azure": - suggested_region = suggested_compute.split("-")[1] - else: - suggested_region = "-".join(suggested_compute.split("-")[1:4]) - suggested_instance = "-".join(suggested_compute.split("-")[-3:-1]) - suggested_size = suggested_compute.split("-")[-1] - return suggested_vendor, suggested_region, suggested_instance, suggested_size - - @property - def tokenizer(self): - return self._tokenizer - - @property - def add_special_tokens(self): - return self._add_special_tokens - - @property - def disable_tqdm(self) -> bool: - return False # no accelerator = this is the main process - - def cleanup(self): - if self.endpoint is not None: - if not self.reuse_existing: - self.endpoint.delete() - logger.warning( - "We deleted the spinned up endpoint after using it. You'll need to create it again if you need to reuse it." - ) - - @property - def max_length(self): - if self._max_length is not None: - return self._max_length - - if hasattr(self.tokenizer, "model_max_length"): - self._max_length = self.tokenizer.model_max_length - else: - self._max_length = 2048 - return self._max_length - - def _async_process_request( - self, - context: str, - stop_tokens: list[str] | None, - max_tokens: int | None, - grammar: Optional[TextGenerationInputGrammarType] = None, - ) -> Coroutine[None, list[TextGenerationOutput], str]: - # Todo: add an option to launch with conversational instead for chat prompts - # https://huggingface.co/docs/huggingface_hub/v0.20.3/en/package_reference/inference_client#huggingface_hub.AsyncInferenceClient.conversational - self.generation_config["grammar"] = grammar - self.generation_config["stop"] = stop_tokens - self.generation_config["max_new_tokens"] = max_tokens - self.generation_config["details"] = True - self.generation_config["decoder_input_details"] = True - - generated_text = self.async_client.text_generation(prompt=context, **self.generation_config) - - return generated_text - - def _process_request( - self, - context: str, - stop_tokens: list[str] | None, - max_tokens: int | None, - grammar: TextGenerationInputGrammarType | None = None, - ) -> TextGenerationOutput: - # Todo: add an option to launch with conversational instead for chat prompts - # https://huggingface.co/docs/huggingface_hub/v0.20.3/en/package_reference/inference_client#huggingface_hub.AsyncInferenceClient.conversational - self.generation_config["stop"] = stop_tokens - self.generation_config["max_new_tokens"] = max_tokens - self.generation_config["details"] = True - self.generation_config["decoder_input_details"] = True - self.generation_config["grammar"] = grammar - - generated_text = self.client.text_generation( - prompt=context, - **self.generation_config, - ) - - return generated_text - - async def _async_process_batch_generate( - self, - docs: list[Doc], - ): - return await asyncio.gather( - *[ - self._async_process_request( - context=self.prompt_manager.prepare_prompt(doc), - stop_tokens=doc.stop_sequences, - max_tokens=doc.generation_size, - grammar=doc.generation_grammar, - ) - for doc in docs - ] - ) - - def _process_batch_generate( - self, - docs: list[Doc], - ) -> list[TextGenerationOutput]: - return [ - self._process_request( - context=self.prompt_manager.prepare_prompt(doc), - stop_tokens=doc.stop_sequences, - max_tokens=doc.generation_size, - grammar=doc.generation_grammar, - ) - for doc in docs - ] - - async def _async_process_batch_logprob(self, docs: list[Doc], rolling: bool = False): - contexts = [self.prompt_manager.prepare_prompt(doc) for doc in docs] - return await asyncio.gather( - *[ - self._async_process_request( - context=context if rolling else context + doc.choices[0], - stop_tokens=[], - max_tokens=1, - grammar=doc.generation_grammar, - ) - for context, doc in zip(contexts, docs) - ] - ) - - def _process_batch_logprob(self, docs: list[Doc], rolling: bool = False) -> list[TextGenerationOutput]: - contexts = [self.prompt_manager.prepare_prompt(doc) for doc in docs] - return [ - self._process_request( - context=context if rolling else context + doc.choices[0], - stop_tokens=[], - max_tokens=1, - grammar=doc.generation_grammar, - ) - for context, doc in zip(contexts, docs) - ] - - @cached(SamplingMethod.GENERATIVE) - def greedy_until( - self, - docs: List[Doc], - ) -> List[ModelResponse]: - return self._greedy_until(docs) - - def _greedy_until(self, docs: List[Doc]) -> list[ModelResponse]: - dataset = GenerativeTaskDataset(requests=docs, num_dataset_splits=self.DATASET_SPLITS) - batch_size = self.config.batch_size - results = [] - - for split in tqdm( - dataset.splits_iterator(), - total=dataset.num_dataset_splits, - desc="Splits", - position=0, - disable=self.disable_tqdm, - ): - dataloader = DataLoader(split, batch_size=batch_size, collate_fn=lambda batch: batch) - - for batch in tqdm( - dataloader, desc="Greedy generation", position=1, leave=False, disable=self.disable_tqdm - ): - num_samples = batch[0].num_samples - if num_samples > 1: - logger.error( - "Inference endpoints does not allow sampling evaluations - this is likely to fail or provide problematic results" - ) - - if self.use_async: - responses = asyncio.run(self._async_process_batch_generate(batch)) - else: - responses = self._process_batch_generate(batch) - for response in responses: - results.append( - ModelResponse( - text=[response.generated_text], - output_tokens=[[token.id for token in response.details.tokens]], - ) - ) - - return dataset.get_original_order(results) - - @cached(SamplingMethod.LOGPROBS) - def loglikelihood(self, docs: list[Doc]) -> list[ModelResponse]: - return self._loglikelihood(docs, rolling=False) - - @cached(SamplingMethod.PERPLEXITY) - def loglikelihood_rolling(self, docs: list[Doc], override_bs=None) -> list[ModelResponse]: - return self._loglikelihood(docs, rolling=True) - - def _loglikelihood(self, docs: list[Doc], rolling: bool = False) -> list[ModelResponse]: - dataset = LoglikelihoodDataset(requests=docs, num_dataset_splits=self.DATASET_SPLITS) - batch_size = self.config.batch_size - results: list[ModelResponse] = [] - - for split in tqdm( - dataset.splits_iterator(), - total=dataset.num_dataset_splits, - desc="Splits", - position=0, - disable=self.disable_tqdm, - ): - dataloader = DataLoader(split, batch_size=batch_size, collate_fn=lambda batch: batch) - - for batch in tqdm( - dataloader, - desc=f"Loglikelihoods {'rolling' if rolling else ''}", - position=1, - leave=False, - disable=self.disable_tqdm, - ): - if self.use_async: - responses = asyncio.run(self._async_process_batch_logprob(batch, rolling=rolling)) - else: - responses = self._process_batch_logprob(batch, rolling=rolling) - - for cur_request, response in zip(batch, responses): - cont_toks = torch.tensor(cur_request.tokenized_continuation) - len_choice = len(cont_toks) - - if rolling: - logits = [t.logprob for t in response.details.tokens[:-1]] - results.append( - ModelResponse( - result=sum(logits), - input_tokens=[t.id for t in response.details.prefill], - generated_tokens=[t.id for t in response.details.tokens[:-1]], - truncated_tokens_count=-1, - padded_tokens_count=-1, - ) - ) - - else: - if self.endpoint: # inference endpoint - logits = [ - t.logprob for t in response.details.prefill[-len_choice:] if t.logprob is not None - ] # to check - else: # serverless endpoint - logits = [ - t.logprob for t in response.details.tokens[-len_choice:] if t.logprob is not None - ] - - greedy_tokens = torch.tensor(logits).argmax(dim=-1) - max_equal = (greedy_tokens == cont_toks).all().squeeze(0) - results.append( - ModelResponse( - logprobs=(sum(logits), bool(max_equal)), - input_tokens=[t.id for t in response.details.prefill[:-len_choice]], - output_tokens=[t.id for t in response.details.prefill[-len_choice:]], - truncated_tokens_count=-1, - padded_tokens_count=-1, - ) - ) - - return dataset.get_original_order(results) diff --git a/src/lighteval/models/endpoints/inference_providers_model.py b/src/lighteval/models/endpoints/inference_providers_model.py deleted file mode 100644 index 54790e45b..000000000 --- a/src/lighteval/models/endpoints/inference_providers_model.py +++ /dev/null @@ -1,269 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import asyncio -import logging -from typing import Any, List, Optional - -from huggingface_hub import AsyncInferenceClient, ChatCompletionOutput -from huggingface_hub.errors import HfHubHTTPError -from pydantic import NonNegativeInt -from tqdm import tqdm -from tqdm.asyncio import tqdm as async_tqdm -from transformers import AutoTokenizer - -from lighteval.data import GenerativeTaskDataset -from lighteval.models.abstract_model import LightevalModel, ModelConfig -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.prompt_manager import PromptManager -from lighteval.tasks.requests import Doc, SamplingMethod -from lighteval.utils.cache_management import SampleCache, cached - - -logger = logging.getLogger(__name__) - - -class InferenceProvidersModelConfig(ModelConfig): - """Configuration class for HuggingFace's inference providers (like Together AI, Anyscale, etc.). - - inference providers doc: https://huggingface.co/docs/inference-providers/en/index - - Attributes: - model_name (str): - Name or identifier of the model to use. - provider (str): - Name of the inference provider. Examples: "together", "anyscale", "runpod", etc. - timeout (int | None): - Request timeout in seconds. If None, uses provider default. - proxies (Any | None): - Proxy configuration for requests. Can be a dict or proxy URL string. - org_to_bill (str | None): - Organization to bill for API usage. If None, bills the user's account. - parallel_calls_count (NonNegativeInt): - Number of parallel API calls to make. Defaults to 10. - Higher values increase throughput but may hit rate limits. - generation_parameters (GenerationParameters, optional, defaults to empty GenerationParameters): - Configuration parameters that control text generation behavior, including - temperature, top_p, max_new_tokens, etc. - system_prompt (str | None, optional, defaults to None): Optional system prompt to be used with chat models. - This prompt sets the behavior and context for the model during evaluation. - cache_dir (str, optional, defaults to "~/.cache/huggingface/lighteval"): Directory to cache the model. - - Example: - ```python - config = InferenceProvidersModelConfig( - model_name="deepseek-ai/DeepSeek-R1-0528", - provider="together", - parallel_calls_count=5, - generation_parameters=GenerationParameters( - temperature=0.7, - max_new_tokens=100 - ) - ) - ``` - - Note: - - Requires HF API keys to be set in environment variable - - Different providers have different rate limits and pricing - """ - - model_name: str - provider: str - timeout: int | None = None - proxies: Any | None = None - org_to_bill: str | None = None - parallel_calls_count: NonNegativeInt = 10 - - -class InferenceProvidersClient(LightevalModel): - """Client for making inference requests to various providers using the HuggingFace Inference API. - - This class handles batched generation requests with automatic retries and error handling. - API keys should be set in environment variables. - """ - - def __init__(self, config: InferenceProvidersModelConfig) -> None: - """Initialize the inference client. - - Args: - config: Configuration object containing model and provider settings - """ - self.config = config - self.model_name = config.model_name - self.provider = config.provider - self.generation_parameters = config.generation_parameters - - self.API_MAX_RETRY = 5 - self.API_RETRY_SLEEP = 3 - self.API_RETRY_MULTIPLIER = 2 - self.pairwise_tokenization = False - self.parallel_calls_count = config.parallel_calls_count - - self.client = AsyncInferenceClient( - provider=self.provider, - timeout=config.timeout, - proxies=config.proxies, - bill_to=config.org_to_bill, - ) - try: - self._tokenizer = AutoTokenizer.from_pretrained(self.model_name) - except HfHubHTTPError: - logger.warning(f"Could not load model's tokenizer for the model {self.model_name}.") - self._tokenizer = None - - self.prompt_manager = PromptManager( - use_chat_template=True, tokenizer=self.tokenizer, system_prompt=config.system_prompt - ) - - # Initialize cache for tokenization and predictions - self._cache = SampleCache(config) - - async def __call_api(self, prompt: List[dict], num_samples: int) -> Optional[ChatCompletionOutput]: - """Make API call with exponential backoff retry logic. - - Args: - prompt: List of message dictionaries for chat completion - num_samples: Number of completions to generate - - Returns: - API response or None if all retries failed - """ - for attempt in range(self.API_MAX_RETRY): - try: - kwargs = { - "model": self.model_name, - "messages": prompt, - "n": num_samples, - } - kwargs.update(self.generation_parameters.to_inference_providers_dict()) - if kwargs.get("temperature") == 0.0 and num_samples > 1: - raise ValueError( - "Temperature is set to 0.0, but num_samples > 1. " - "This is not supported by the inference providers API." - ) - response: ChatCompletionOutput = await self.client.chat.completions.create(**kwargs) - return response - except Exception as e: - wait_time = min(64, self.API_RETRY_SLEEP * (2**attempt)) # Exponential backoff with max 64s - logger.warning( - f"Error in API call: {e}, waiting {wait_time} seconds before retry {attempt + 1}/{self.API_MAX_RETRY}" - ) - await asyncio.sleep(wait_time) - - logger.error(f"API call failed after {self.API_MAX_RETRY} attempts, returning empty response.") - return None - - async def __call_api_parallel( - self, - prompts, - num_samples: int | list[int], - ): - results = [] - - # Initialize semaphore for the current event loop - semaphore = asyncio.Semaphore(self.parallel_calls_count) - - num_sampless = [num_samples for _ in prompts] if not isinstance(num_samples, list) else num_samples - assert len(prompts) == len(num_sampless), ( - f"Length of prompts and max_new_tokenss should be the same but are {len(prompts)}, {len(num_sampless)}" - ) - - async def bounded_api_call(prompt, num_samples): - async with semaphore: - return await self.__call_api(prompt, num_samples) - - tasks = [bounded_api_call(prompt, num_samples) for prompt, num_samples in zip(prompts, num_sampless)] - results = await async_tqdm.gather(*tasks) - - if None in results: - raise ValueError("Some entries are not annotated due to errors in __call_api, please inspect and retry.") - - return results - - @cached(SamplingMethod.GENERATIVE) - def greedy_until( - self, - docs: list[Doc], - ) -> list[ModelResponse]: - """Generates responses using a greedy decoding strategy until certain ending conditions are met. - - Args: - docs (list[Doc]): List of documents containing the context for generation. - - Returns: - list[ModelResponse]: list of generated responses. - """ - dataset = GenerativeTaskDataset(requests=docs, num_dataset_splits=self.DATASET_SPLITS) - results = [] - - for split in tqdm( - dataset.splits_iterator(), - total=dataset.num_dataset_splits, - desc="Splits", - position=0, - disable=False, # self.disable_tqdm, - ): - contexts = [self.prompt_manager.prepare_prompt_api(doc) for doc in split] - num_samples = split[0].num_samples - - responses = asyncio.run(self.__call_api_parallel(contexts, num_samples)) - - for response, context in zip(responses, contexts): - result: list[str] = [choice.message.content for choice in response.choices] - - cur_response = ModelResponse( - # In empty responses, the model should return an empty string instead of None - text=result if result[0] else [""], - input=context, - ) - results.append(cur_response) - - return dataset.get_original_order(results) - - @property - def tokenizer(self): - return self._tokenizer - - @property - def add_special_tokens(self) -> bool: - return False - - @property - def max_length(self) -> int: - """Return the maximum sequence length of the model.""" - try: - return self._tokenizer.model_max_length - except AttributeError: - logger.warning("Tokenizer was not correctly loaded. Max model context length is assumed to be 30K tokens") - return 30000 - - @cached(SamplingMethod.LOGPROBS) - def loglikelihood(self, docs: list[Doc]) -> list[ModelResponse]: - """Tokenize the context and continuation and compute the log likelihood of those - tokenized sequences. - """ - raise NotImplementedError - - @cached(SamplingMethod.PERPLEXITY) - def loglikelihood_rolling(self, docs: list[Doc]) -> list[ModelResponse]: - """This function is used to compute the log likelihood of the context for perplexity metrics.""" - raise NotImplementedError diff --git a/src/lighteval/models/endpoints/litellm_model.py b/src/lighteval/models/endpoints/litellm_model.py deleted file mode 100644 index 87332d1d7..000000000 --- a/src/lighteval/models/endpoints/litellm_model.py +++ /dev/null @@ -1,416 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import logging -import time -from concurrent.futures import ThreadPoolExecutor -from json import JSONDecodeError - -import requests -from tqdm import tqdm - -from lighteval.data import GenerativeTaskDataset -from lighteval.models.abstract_model import LightevalModel, ModelConfig -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.prompt_manager import PromptManager -from lighteval.tasks.requests import Doc, SamplingMethod -from lighteval.utils.cache_management import SampleCache, cached -from lighteval.utils.imports import is_package_available, requires - - -logger = logging.getLogger(__name__) - -if is_package_available("litellm"): - import litellm - from litellm import encode, supports_reasoning - from litellm.caching.caching import Cache, LiteLLMCacheType - from litellm.utils import ModelResponse as LitellmModelResponse - from litellm.utils import get_max_tokens - - logging.getLogger("LiteLLM").setLevel(logging.WARNING) - logging.getLogger("LiteLLM").handlers.clear() - - litellm.cache = Cache(type=LiteLLMCacheType.DISK) -else: - from unittest.mock import Mock - - litellm = Mock() - encode = Mock() - LitellmModelResponse = Mock() - - -class LiteLLMModelConfig(ModelConfig): - """Configuration class for LiteLLM unified API client. - - This configuration is used to connect to various LLM providers through the LiteLLM - unified API. LiteLLM provides a consistent interface to multiple providers including - OpenAI, Anthropic, Google, and many others. - - litellm doc: https://docs.litellm.ai/docs/ - - Attributes: - model_name (str): - Model identifier. Can include provider prefix (e.g., "gpt-4", "claude-3-sonnet") - or use provider/model format (e.g., "openai/gpt-4", "anthropic/claude-3-sonnet"). - provider (str | None): - Optional provider name override. If None, inferred from model_name. - Examples: "openai", "anthropic", "google", "cohere", etc. - base_url (str | None): - Custom base URL for the API. If None, uses provider's default URL. - Useful for using custom endpoints or local deployments. - api_key (str | None): - API key for authentication. If None, reads from environment variables. - Environment variable names are provider-specific (e.g., OPENAI_API_KEY). - concurrent_requests (int): - Maximum number of concurrent API requests to execute in parallel. - Higher values can improve throughput for batch processing but may hit rate limits - or exhaust API quotas faster. Default is 10. - verbose (bool): - Whether to enable verbose logging. Default is False. - max_model_length (int | None): - Maximum context length for the model. If None, infers the model's default max length. - api_max_retry (int): - Maximum number of retries for API requests. Default is 8. - api_retry_sleep (float): - Initial sleep time (in seconds) between retries. Default is 1.0. - api_retry_multiplier (float): - Multiplier for increasing sleep time between retries. Default is 2.0. - timeout (float): - Request timeout in seconds. Default is None (no timeout). - generation_parameters (GenerationParameters, optional, defaults to empty GenerationParameters): - Configuration parameters that control text generation behavior, including - temperature, top_p, max_new_tokens, etc. - system_prompt (str | None, optional, defaults to None): Optional system prompt to be used with chat models. - This prompt sets the behavior and context for the model during evaluation. - cache_dir (str, optional, defaults to "~/.cache/huggingface/lighteval"): Directory to cache the model. - - Example: - ```python - config = LiteLLMModelConfig( - model_name="gpt-4", - provider="openai", - base_url="https://api.openai.com/v1", - concurrent_requests=5, - generation_parameters=GenerationParameters( - temperature=0.7, - max_new_tokens=100 - ) - ) - ``` - """ - - model_name: str - provider: str | None = None - base_url: str | None = None - api_key: str | None = None - concurrent_requests: int = 10 - verbose: bool = False - max_model_length: int | None = None - - api_max_retry: int = 8 - api_retry_sleep: float = 1.0 - api_retry_multiplier: float = 2.0 - timeout: float | None = None - - -@requires("litellm") -class LiteLLMClient(LightevalModel): - _DEFAULT_MAX_LENGTH: int = 4096 - - def __init__(self, config: LiteLLMModelConfig) -> None: - """IMPORTANT: Your API keys should be set in the environment variables. - If a base_url is not set, it will default to the public API. - """ - self.config = config - self.model = config.model_name - self.provider = config.provider or config.model_name.split("/")[0] - self.base_url = config.base_url - self.api_key = config.api_key - self.generation_parameters = config.generation_parameters - self.concurrent_requests = config.concurrent_requests - self._max_length = config.max_model_length - - self.API_MAX_RETRY = config.api_max_retry - self.API_RETRY_SLEEP = config.api_retry_sleep - self.API_RETRY_MULTIPLIER = config.api_retry_multiplier - self.timeout = config.timeout - - self._tokenizer = encode - self.pairwise_tokenization = False - litellm.drop_params = True - litellm.verbose = config.verbose - self.prompt_manager = PromptManager( - use_chat_template=True, tokenizer=self.tokenizer, system_prompt=config.system_prompt - ) - - # Initialize cache for tokenization and predictions - self._cache = SampleCache(config) - - def _prepare_stop_sequence(self, stop_sequence): - """Prepare and validate stop sequence.""" - if self.provider == "anthropic": - # Filter out whitespace-only stop sequences - if stop_sequence: - stop_sequence = [s for s in stop_sequence if s and s.strip()] - return stop_sequence - - def _prepare_max_new_tokens(self, max_new_tokens) -> int | None: - """Calculate completion tokens based on max_new_tokens.""" - if not max_new_tokens or max_new_tokens <= 0: - return None - - if supports_reasoning(self.model): - # We need to allow more tokens to include reasoning tokens - max_new_tokens = min(max_new_tokens * 10, self.max_length) - - logger.warning( - f"Reasoning model detected, increasing max_new_tokens to {max_new_tokens} to allow for reasoning tokens", - ) - - return max_new_tokens - - def __call_api(self, prompt, return_logits, max_new_tokens, num_samples, stop_sequence): # noqa: C901 - """Make API call with retries.""" - response = LitellmModelResponse() - stop_sequence = self._prepare_stop_sequence(stop_sequence) - max_new_tokens = self._prepare_max_new_tokens(max_new_tokens) - - if return_logits and not self.provider == "openai": - logger.warning("Returning logits is not supported for this provider, ignoring.") - - # Prepare kwargs for completion call - kwargs = { - "model": self.model, - "messages": prompt, - "response_format": {"type": "text"}, - "max_tokens": max_new_tokens, - "logprobs": return_logits if self.provider == "openai" else None, - "stop": stop_sequence, - "base_url": self.base_url, - "api_key": self.api_key, - "n": num_samples, - "caching": True, - "timeout": self.timeout, - } - - if "o1" in self.model: - logger.warning("O1 models do not support temperature, top_p, stop sequence. Disabling.") - else: - kwargs.update(self.generation_parameters.to_litellm_dict()) - - if kwargs.get("max_completion_tokens", None) is None: - kwargs["max_completion_tokens"] = max_new_tokens - - for attempt in range(self.API_MAX_RETRY): - try: - response = litellm.completion(**kwargs) - content = response.choices[0].message.content - - # If response is empty, retry without caching (maybe the error is recoverable and solved with a retry) - if not content: - logger.info("Response is empty, retrying without caching") - kwargs["caching"] = False - response = litellm.completion(**kwargs) - content = response.choices[0].message.content - - return response - except litellm.BadRequestError as e: - if "message" in e.__dict__: - error_string = ( - "The response was filtered due to the prompt triggering Microsoft's content management policy" - ) - if error_string in e.__dict__["message"]: - logger.warning(f"{error_string}. Returning empty response.") - return LitellmModelResponse() - except Exception as e: - wait_time = min( - 64, self.API_RETRY_SLEEP * (self.API_RETRY_MULTIPLIER**attempt) - ) # Exponential backoff with max 64s - logger.warning( - f"Error in API call: {e}, waiting {wait_time} seconds before retry {attempt + 1}/{self.API_MAX_RETRY}" - ) - time.sleep(wait_time) - - logger.error(f"API call failed after {self.API_MAX_RETRY} attempts, returning empty response.") - return LitellmModelResponse() - - def __call_api_parallel( - self, - prompts, - return_logits: bool | list[bool], - max_new_tokens: int | list[int] | None, - num_samples: int | list[int], - stop_sequence: list[str] | None = None, - ): - results = [] - - return_logitss = [return_logits for _ in prompts] if not isinstance(return_logits, list) else return_logits - max_new_tokenss = [max_new_tokens for _ in prompts] if not isinstance(max_new_tokens, list) else max_new_tokens - num_sampless = [num_samples for _ in prompts] if not isinstance(num_samples, list) else num_samples - stop_sequencess = [stop_sequence for _ in prompts] - assert ( - len(prompts) == len(return_logitss) == len(max_new_tokenss) == len(num_sampless) == len(stop_sequencess) - ), ( - f"Length of prompts, return_logitss, max_new_tokenss, num_sampless, stop_sequences, system_prompts should be the same but are {len(prompts)}, {len(return_logitss)}, {len(max_new_tokenss)}, {len(num_sampless)}, {len(stop_sequencess)}" - ) - - with ThreadPoolExecutor(self.concurrent_requests) as executor: - for entry in tqdm( - executor.map( - self.__call_api, - prompts, - return_logitss, - max_new_tokenss, - num_sampless, - stop_sequencess, - ), - total=len(prompts), - ): - results.append(entry) - - if None in results: - raise ValueError("Some entries are not annotated due to errors in annotate_p, please inspect and retry.") - - return results - - def estimate_context_length(self) -> int: - def fallback(): - logger.warning("Failed to fetch model endpoint info from OpenRouter, returning default max length.") - return self._DEFAULT_MAX_LENGTH - - # If the model is used through openrouter, the actual model name comes after the prefix - model_name = self.model.removeprefix("openrouter/") - endpoint_info_response = requests.get( - f"https://openrouter.ai/api/v1/models/{model_name}/endpoints", - headers={}, - ) - if endpoint_info_response.ok: - try: - endpoint_info = endpoint_info_response.json() - context_lengths = { - endpoint["provider_name"]: endpoint["context_length"] - for endpoint in endpoint_info["data"]["endpoints"] - } - - if self.provider in context_lengths: - return context_lengths[self.provider] - - min_length = min(context_lengths.values()) - logger.warning( - f"Estimating model context length as the minimum context length from available OpenRouter providers: {min_length}" - ) - return min_length - except (KeyError, TypeError, ValueError, JSONDecodeError): - return fallback() - - return fallback() - - @cached(SamplingMethod.GENERATIVE) - def greedy_until( - self, - docs: list[Doc], - ) -> list[ModelResponse]: - """Generates responses using a greedy decoding strategy until certain ending conditions are met. - - Args: - docs (list[Doc]): List of documents containing the context for generation. - - Returns: - list[ModelResponse]: list of generated responses. - """ - dataset = GenerativeTaskDataset(requests=docs, num_dataset_splits=self.DATASET_SPLITS) - results = [] - - for split in tqdm( - dataset.splits_iterator(), - total=dataset.num_dataset_splits, - desc="Splits", - position=0, - disable=self.disable_tqdm, - ): - contexts = [self.prompt_manager.prepare_prompt_api(doc) for doc in dataset] - max_new_tokens = split[0].generation_size # could be none - return_logits = split[0].use_logits - num_samples = split[0].num_samples - stop_sequence = split[0].stop_sequences - - if num_samples > 1 and self.generation_parameters.temperature == 0: - raise ValueError( - "num_samples > 1 is not supported with temperature=0, please set temperature > 0 or use non sampling metrics." - ) - - responses = self.__call_api_parallel(contexts, return_logits, max_new_tokens, num_samples, stop_sequence) - - for response, context in zip(responses, contexts): - result: list[str] = [choice.message.content for choice in response.choices] - reasonings: list[str | None] = [ - getattr(choice.message, "reasoning_content", None) for choice in response.choices - ] - - cur_response = ModelResponse( - # In empty responses, the model should return an empty string instead of None - text=result if result[0] else [""], - reasonings=reasonings, - input=context, - ) - results.append(cur_response) - - return dataset.get_original_order(results) - - @property - def tokenizer(self): - return self._tokenizer - - @property - def add_special_tokens(self) -> bool: - return False - - @property - def max_length(self) -> int: - """Return the maximum sequence length of the model.""" - if self._max_length is not None: - return self._max_length - - try: - max_tokens = get_max_tokens(self.model) - except Exception: - logger.error( - f"Unable to get the maximum sequence length for model {self.model} from litellm. Fetching information from OpenRouter instead." - ) - max_tokens = self.estimate_context_length() - - # Avoid future requests - self._max_length = max_tokens - - return max_tokens - - @cached(SamplingMethod.LOGPROBS) - def loglikelihood(self, docs: list[Doc]) -> list[ModelResponse]: - """Tokenize the context and continuation and compute the log likelihood of those - tokenized sequences. - """ - raise NotImplementedError - - @cached(SamplingMethod.PERPLEXITY) - def loglikelihood_rolling(self, docs: list[Doc]) -> list[ModelResponse]: - """This function is used to compute the log likelihood of the context for perplexity metrics.""" - raise NotImplementedError diff --git a/src/lighteval/models/endpoints/tgi_model.py b/src/lighteval/models/endpoints/tgi_model.py deleted file mode 100644 index 4fd765b8d..000000000 --- a/src/lighteval/models/endpoints/tgi_model.py +++ /dev/null @@ -1,202 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import asyncio -from dataclasses import replace -from typing import Coroutine, Optional - -import requests -from huggingface_hub import TextGenerationInputGenerateParameters, TextGenerationInputGrammarType, TextGenerationOutput -from transformers.models.auto.tokenization_auto import AutoTokenizer - -from lighteval.models.abstract_model import ModelConfig -from lighteval.models.endpoints.endpoint_model import InferenceEndpointModel -from lighteval.tasks.prompt_manager import PromptManager -from lighteval.utils.cache_management import SampleCache -from lighteval.utils.imports import Extra, is_package_available, requires - - -if is_package_available(Extra.TGI): - from text_generation import AsyncClient -else: - from unittest.mock import Mock - - AsyncClient = Mock() - - -BATCH_SIZE = 50 - - -def divide_chunks(array, n): - # looping till length array - for i in range(0, len(array), n): - yield array[i : i + n] - - -class TGIModelConfig(ModelConfig): - """Configuration class for Text Generation Inference (TGI) backend. - - doc: https://huggingface.co/docs/text-generation-inference/en/index - - This configuration is used to connect to TGI servers that serve HuggingFace models - using the text-generation-inference library. TGI provides high-performance inference - with features like continuous batching and efficient memory management. - - Attributes: - inference_server_address (str | None): - Address of the TGI server. Format: "http://host:port" or "https://host:port". - Example: "http://localhost:8080" - inference_server_auth (str | None): - Authentication token for the TGI server. If None, no authentication is used. - model_name (str | None): - Optional model name override. If None, uses the model name from server info. - generation_parameters (GenerationParameters, optional, defaults to empty GenerationParameters): - Configuration parameters that control text generation behavior, including - temperature, top_p, max_new_tokens, etc. - system_prompt (str | None, optional, defaults to None): Optional system prompt to be used with chat models. - This prompt sets the behavior and context for the model during evaluation. - cache_dir (str, optional, defaults to "~/.cache/huggingface/lighteval"): Directory to cache the model. - - Example: - ```python - config = TGIModelConfig( - inference_server_address="http://localhost:8080", - inference_server_auth="your-auth-token", - model_name="meta-llama/Llama-3.1-8B-Instruct", - generation_parameters=GenerationParameters( - temperature=0.7, - max_new_tokens=100 - ) - ) - ``` - """ - - inference_server_address: str | None = None - inference_server_auth: str | None = None - model_name: str | None - model_info: dict | None = None - batch_size: int = 1 - - -# inherit from InferenceEndpointModel instead of LightevalModel since they both use the same interface, and only overwrite -# the client functions, since they use a different client. -class ModelClient(InferenceEndpointModel): - _DEFAULT_MAX_LENGTH: int = 4096 - - def __init__(self, config: TGIModelConfig) -> None: - headers = ( - {} if config.inference_server_auth is None else {"Authorization": f"Bearer {config.inference_server_auth}"} - ) - - self.client = AsyncClient(config.inference_server_address, headers=headers, timeout=240) - self.generation_parameters = config.generation_parameters - self.generation_config = TextGenerationInputGenerateParameters(**self.generation_parameters.to_tgi_ie_dict()) - self._max_gen_toks = 256 - self.model_info = requests.get(f"{config.inference_server_address}/info", headers=headers).json() - if "model_id" not in self.model_info: - raise ValueError("Error occurred when fetching info: " + str(self.model_info)) - if config.model_name: - self.model_info["model_id"] = config.model_name - else: - # Set the model_name in config to the actual model_id from server for caching - config.model_name = self.model_info["model_id"] - self.config = config - self._tokenizer = AutoTokenizer.from_pretrained(self.model_info["model_id"]) - self._add_special_tokens = True - self.use_async = True - self.config.model_info = self.model_info - - # Initialize prompt manager (required by parent class) - self.prompt_manager = PromptManager( - use_chat_template=True, tokenizer=self.tokenizer, system_prompt=config.system_prompt - ) - - # Initialize cache for tokenization and predictions - self._cache = SampleCache(config) - - @requires(Extra.TGI) - def _async_process_request( - self, - context: str, - stop_tokens: list[str], - max_tokens: int, - grammar: Optional[TextGenerationInputGrammarType] = None, - ) -> Coroutine[None, list[TextGenerationOutput], str]: - # Todo: add an option to launch with conversational instead for chat prompts - # We create a copy of the current text generation params - generation_config: TextGenerationInputGenerateParameters = replace( - self.generation_config, - stop=stop_tokens, - max_new_tokens=max_tokens, - details=True, - decoder_input_details=True, - grammar=grammar, - ) - - generated_text = self.client.generate( - prompt=context, - do_sample=generation_config.do_sample or False, - max_new_tokens=generation_config.max_new_tokens, - best_of=generation_config.best_of, - repetition_penalty=generation_config.repetition_penalty, - return_full_text=generation_config.return_full_text or False, - seed=generation_config.seed, - stop_sequences=generation_config.stop, - temperature=generation_config.temperature, - top_k=generation_config.top_k, - top_p=generation_config.top_p, - truncate=generation_config.truncate, - typical_p=generation_config.typical_p, - watermark=generation_config.watermark or False, - decoder_input_details=generation_config.decoder_input_details, - grammar=generation_config.grammar, - ) - - return generated_text - - @requires(Extra.TGI) - def _process_request(self, *args, **kwargs) -> TextGenerationOutput: - return asyncio.run(self._async_process_request(*args, **kwargs)) - - def set_cache_hook(self, cache_hook): - self.cache_hook = cache_hook - - @property - def tokenizer(self): - return self._tokenizer - - @property - def add_special_tokens(self): - return self._add_special_tokens - - @property - def max_length(self) -> int: - if hasattr(self.tokenizer, "model_max_length"): - return self.tokenizer.model_max_length - return ModelClient._DEFAULT_MAX_LENGTH - - @property - def disable_tqdm(self) -> bool: - return False - - def cleanup(self): - pass diff --git a/src/lighteval/models/model_input.py b/src/lighteval/models/model_input.py deleted file mode 100644 index ad41c23eb..000000000 --- a/src/lighteval/models/model_input.py +++ /dev/null @@ -1,243 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -from pydantic import BaseModel, NonNegativeFloat, NonNegativeInt - - -class GenerationParameters(BaseModel, extra="forbid"): - num_blocks: NonNegativeInt | None = None # transformers - block_size: NonNegativeInt | None = None # transformers - - early_stopping: bool | None = None # transformers - repetition_penalty: NonNegativeFloat | None = None # vllm, transformers, tgi, sglang - frequency_penalty: NonNegativeFloat | None = None # vllm, tgi, sglang - length_penalty: NonNegativeFloat | None = None # vllm, transformers - presence_penalty: NonNegativeFloat | None = None # vllm, sglang - - max_new_tokens: NonNegativeInt | None = None # vllm, transformers, tgi, litellm, sglang - min_new_tokens: NonNegativeInt | None = None # vllm, transformers, sglang - - seed: NonNegativeInt | None = None # vllm, tgi, litellm - stop_tokens: list[str] | None = None # vllm, transformers, tgi, litellm, sglang - temperature: NonNegativeFloat = ( - 0 # vllm, transformers, tgi, litellm, sglang # if not set, defaults to greedy decoding - ) - top_k: NonNegativeInt | None = None # vllm, transformers, tgi, sglang - min_p: NonNegativeFloat | None = None # vllm, transformers, sglang - top_p: NonNegativeFloat | None = None # vllm, transformers, tgi, litellm, sglang - truncate_prompt: bool | None = None # vllm, tgi - - cache_implementation: str | None = None # transformers - - # response format to be followed by the model, - # more info here https://platform.openai.com/docs/api-reference/chat/create#chat-create-response_format - response_format: str | None = None # inference_providers - - @classmethod - def from_dict(cls, config_dict: dict): - """Creates a GenerationParameters object from a config dictionary - - Args: - config_dict (dict): Config dictionary. Must obey the following shape: - {"generation": - { - "early_stopping": value, - ... - "truncate_prompt": value - } - } - - Returns: - GenerationParameters: A GenerationParameters object created from the config dictionary - """ - return GenerationParameters(**config_dict.get("generation", {})) - - @classmethod - def from_model_args(cls, model_args: str): - """Creates a GenerationParameters object from a model_args string. - - It's used when the model_args are passed as a string in the command line. - The generation parameters must follow the following format (at any place in the string): - "generation_parameters={key1:value1,key2=value2}" - - Args: - model_args (str): A string like the following: - "pretrained=deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B,dtype=float16,max_model_length=32768,generation_parameters={temperature:0.7,top_p:5}" - - Returns: - GenerationParameters: A GenerationParameters object created from the model args string - """ - - def parse_model_args(model_args): - import json - import re - - pattern = re.compile(r"(\w+)=(\{.*\}|[^,]+)") - matches = pattern.findall(model_args) - for key, value in matches: - key = key.strip() - if key == "generation_parameters": - gen_params = re.sub(r"(\w+):", r'"\1":', value) - return json.loads(gen_params) - - params: dict = parse_model_args(model_args) or {} - return GenerationParameters(**params) - - def to_litellm_dict(self) -> dict: - """Selects relevant generation and sampling parameters for litellm models. - Doc: https://docs.litellm.ai/docs/completion/input#input-params-1 - - Returns: - dict: The parameters to create a litellm.SamplingParams in the model config. - """ - args = { - "max_completion_tokens": self.max_new_tokens, - "stop": self.stop_tokens, - "temperature": self.temperature, - "top_p": self.top_p, - "seed": self.seed, - "repetition_penalty": self.repetition_penalty, - "frequency_penalty": self.frequency_penalty, - } - return {k: v for k, v in args.items() if v is not None} - - def to_inference_providers_dict(self) -> dict: - """Selects relevant generation and sampling parameters for litellm models. - Doc: https://docs.litellm.ai/docs/completion/input#input-params-1 - - Returns: - dict: The parameters to create a litellm.SamplingParams in the model config. - """ - args = { - "top_p": self.top_p, - "temperature": self.temperature, - "stop": self.stop_tokens, - "seed": self.seed, - "response_format": self.response_format, - "presence_penalty": self.presence_penalty, - "max_tokens": self.max_new_tokens, - "logprobs": None, - "logit_bias": None, - "frequency_penalty": self.frequency_penalty, - } - return {k: v for k, v in args.items() if v is not None} - - def to_vllm_dict(self) -> dict: - """Selects relevant generation and sampling parameters for vllm models. - Doc: https://docs.vllm.ai/en/v0.5.5/dev/sampling_params.html - - Returns: - dict: The parameters to create a vllm.SamplingParams in the model config. - """ - sampling_params_to_vllm_naming = { - "max_new_tokens": "max_tokens", - "min_new_tokens": "min_tokens", - "stop_tokens": "stop", - } - - # Task specific sampling params to set in model: n, best_of, use_beam_search - # Generation specific params to set in model: logprobs, prompt_logprobs - x = {sampling_params_to_vllm_naming.get(k, k): v for k, v in self.model_dump().items() if v is not None} - # VLLM max_tokens is 16 by default, however the pipeline expect the max_tokens to be None, if the user didn't specify it - if not x.get("max_tokens"): - x["max_tokens"] = None - return x - - def to_vllm_openai_dict(self) -> dict: - """Selects relevant generation and sampling parameters for vllm and openai models. - Doc: https://docs.vllm.ai/en/v0.5.5/dev/sampling_params.html - - Returns: - dict: The parameters to create a vllm.SamplingParams or just provide OpenAI params as such in the model config. - """ - # Task specific sampling params to set in model: n, best_of, use_beam_search - # Generation specific params to set in model: logprobs, prompt_logprobs - return {k: v for k, v in self.model_dump().items() if v is not None} - - def to_transformers_dict(self) -> dict: - """Selects relevant generation and sampling parameters for transformers models. - Doc: https://huggingface.co/docs/transformers/v4.46.3/en/main_classes/text_generation#transformers.GenerationConfig - - Note: We actually don't use the GenerationConfig object itself because it has a huge number of parameters automatically - initialized, to a config which slows down evals insanely. - - Returns: - dict: The parameters to create a transformers.GenerationConfig in the model config. - """ - # Task specific sampling params to set in model: do_sample, num_return_sequences, num_beans - args = { - "max_new_tokens": self.max_new_tokens, - "min_new_tokens": self.min_new_tokens, - "early_stopping": self.early_stopping, - "stop_strings": self.stop_tokens, - "temperature": self.temperature, - "top_k": self.top_k, - "top_p": self.top_p, - "min_p": self.min_p, - "repetition_penalty": self.repetition_penalty, - "length_penalty": self.length_penalty, - "output_scores": True, - "num_blocks": self.num_blocks, - "block_size": self.block_size, - "return_dict_in_generate": True, - "cache_implementation": self.cache_implementation, - } - return {k: v for k, v in args.items() if v is not None} - - def to_tgi_ie_dict(self) -> dict: - """Selects relevant generation and sampling parameters for tgi or inference endpoints models. - Doc: https://huggingface.co/docs/huggingface_hub/v0.26.3/en/package_reference/inference_types#huggingface_hub.TextGenerationInputGenerateParameters - - Returns: - dict: The parameters to create a huggingface_hub.TextGenerationInputGenerateParameters in the model config. - """ - # Task specific sampling params to set in model: best_of, do_sample - args = { - "decoder_input_details": True, - "details": True, - "frequency_penalty": self.frequency_penalty, - "max_new_tokens": self.max_new_tokens, - "repetition_penalty": self.repetition_penalty, - "seed": self.seed, - "stop": self.stop_tokens, - "temperature": self.temperature, - "top_k": self.top_k, - "top_p": self.top_p, - "truncate": self.truncate_prompt, - } - return {k: v for k, v in args.items() if v is not None} - - def to_sglang_dict(self) -> dict: - args = { - "max_new_tokens": self.max_new_tokens, - "temperature": self.temperature, - "stop": self.stop_tokens, - "top_p": self.top_p, - "top_k": self.top_k, - "min_p": self.min_p, - "frequency_penalty": self.frequency_penalty, - "presence_penalty": self.presence_penalty, - "repetition_penalty": self.repetition_penalty, - "min_new_tokens": self.min_new_tokens, - } - return {k: v for k, v in args.items() if v is not None} diff --git a/src/lighteval/models/model_loader.py b/src/lighteval/models/model_loader.py deleted file mode 100644 index e3c710edd..000000000 --- a/src/lighteval/models/model_loader.py +++ /dev/null @@ -1,171 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import logging -from typing import Union - -from lighteval.models.abstract_model import LightevalModel, ModelConfig -from lighteval.models.custom.custom_model import CustomModelConfig -from lighteval.models.dummy.dummy_model import DummyModel, DummyModelConfig -from lighteval.models.endpoints.endpoint_model import ( - InferenceEndpointModel, - InferenceEndpointModelConfig, - ServerlessEndpointModelConfig, -) -from lighteval.models.endpoints.inference_providers_model import ( - InferenceProvidersClient, - InferenceProvidersModelConfig, -) -from lighteval.models.endpoints.litellm_model import LiteLLMClient, LiteLLMModelConfig -from lighteval.models.endpoints.tgi_model import ModelClient, TGIModelConfig -from lighteval.models.sglang.sglang_model import SGLangModel, SGLangModelConfig -from lighteval.models.transformers.adapter_model import AdapterModel, AdapterModelConfig -from lighteval.models.transformers.delta_model import DeltaModel, DeltaModelConfig -from lighteval.models.transformers.transformers_model import TransformersModel, TransformersModelConfig -from lighteval.models.transformers.vlm_transformers_model import VLMTransformersModel, VLMTransformersModelConfig -from lighteval.models.vllm.vllm_model import AsyncVLLMModel, VLLMModel, VLLMModelConfig - - -logger = logging.getLogger(__name__) - - -def load_model( # noqa: C901 - config: ModelConfig, -) -> LightevalModel: - """Load a model from a checkpoint, depending on the config type. - - Args: - config (ModelConfig): configuration of the model to load - - Returns: - LightevalModel: The model that will be evaluated - """ - # Inference server loading - if isinstance(config, TGIModelConfig): - return load_model_with_tgi(config) - - if isinstance(config, InferenceEndpointModelConfig) or isinstance(config, ServerlessEndpointModelConfig): - return load_model_with_inference_endpoints(config) - - if isinstance(config, TransformersModelConfig): - return load_model_with_accelerate_or_default(config) - - if isinstance(config, VLMTransformersModelConfig): - return load_model_with_accelerate_or_default(config) - - if isinstance(config, DummyModelConfig): - return load_dummy_model(config) - - if isinstance(config, VLLMModelConfig): - return load_model_with_accelerate_or_default(config) - - if isinstance(config, CustomModelConfig): - return load_custom_model(config=config) - - if isinstance(config, SGLangModelConfig): - return load_sglang_model(config) - - if isinstance(config, LiteLLMModelConfig): - return load_litellm_model(config) - - if isinstance(config, InferenceProvidersModelConfig): - return load_inference_providers_model(config=config) - - -def load_model_with_tgi(config: TGIModelConfig): - logger.info(f"Load model from inference server: {config.inference_server_address}") - model = ModelClient(config=config) - return model - - -def load_litellm_model(config: LiteLLMModelConfig): - model = LiteLLMClient(config) - return model - - -def load_custom_model(config: CustomModelConfig): - logger.warning(f"Executing custom model code loaded from {config.model_definition_file_path}.") - - import importlib.util - - # Load the Python file - spec = importlib.util.spec_from_file_location("custom_model_module", config.model_definition_file_path) - if spec is None or spec.loader is None: - raise ImportError(f"Could not load file: {config.model_definition_file_path}") - - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - # Find the first class that inherits from LightevalModel - model_class = None - for attr_name in dir(module): - attr = getattr(module, attr_name) - if isinstance(attr, type) and issubclass(attr, LightevalModel) and attr != LightevalModel: - model_class = attr - break - - if model_class is None: - raise ValueError(f"No class inheriting from LightevalModel found in {config.model_definition_file_path}") - - model = model_class(config) - - return model - - -def load_model_with_inference_endpoints(config: Union[InferenceEndpointModelConfig, ServerlessEndpointModelConfig]): - logger.info("Spin up model using inference endpoint.") - model = InferenceEndpointModel(config=config) - return model - - -def load_model_with_accelerate_or_default( - config: Union[ - AdapterModelConfig, TransformersModelConfig, DeltaModelConfig, VLLMModelConfig, VLMTransformersModelConfig - ], -): - if isinstance(config, AdapterModelConfig): - model = AdapterModel(config=config) - elif isinstance(config, DeltaModelConfig): - model = DeltaModel(config=config) - elif isinstance(config, VLLMModelConfig): - if config.is_async: - model = AsyncVLLMModel(config=config) - else: - model = VLLMModel(config=config) - elif isinstance(config, VLMTransformersModelConfig): - model = VLMTransformersModel(config=config) - else: - model = TransformersModel(config=config) - - return model - - -def load_dummy_model(config: DummyModelConfig): - return DummyModel(config=config) - - -def load_inference_providers_model(config: InferenceProvidersModelConfig): - return InferenceProvidersClient(config=config) - - -def load_sglang_model(config: SGLangModelConfig): - return SGLangModel(config=config) diff --git a/src/lighteval/models/model_output.py b/src/lighteval/models/model_output.py deleted file mode 100644 index b10ce7f56..000000000 --- a/src/lighteval/models/model_output.py +++ /dev/null @@ -1,168 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from dataclasses import dataclass, field - -import torch - - -@dataclass -class ModelResponse: - """A class to represent the response from a model during evaluation. - - This dataclass contains all the information returned by a model during inference, - including generated text, log probabilities, token information, and metadata. - Different attributes are required for different types of evaluation metrics. - - Attributes: - input (str | list | None): - The original input prompt or context that was fed to the model. - Used for debugging and analysis purposes. - - input_tokens (list[int]): - The tokenized representation of the input prompt. - Useful for understanding how the model processes the input. - - text (list[str]): - The generated text responses from the model. Each element represents - one generation (useful when num_samples > 1). - **Required for**: Generative metrics, exact match, llm as a judge, etc. - - text_post_processed (Optional[list[str]]): - The generated text responses from the model, but post processed. - Atm, post processing removes thinking/reasoning steps. - - Careful! This is not computed by default, but in a separate step by calling - `post_process` on the ModelResponse object. - **Required for**: Generative metrics that require direct answers. - - logprobs (list[float]): - Log probabilities of the generated tokens or sequences. - **Required for**: loglikelihood and perplexity metrics. - - argmax_logits_eq_gold (list[bool]): - Whether the argmax logits match the gold/expected text. - Used for accuracy calculations in multiple choice and classification tasks. - **Required for**: certain loglikelihood metrics. - - - unconditioned_logprobs (Optional[list[float]]): - Log probabilities from an unconditioned model (e.g., without context). - Used for PMI (Pointwise Mutual Information) normalization. - **Required for**: PMI metrics. - - Usage Examples: - - **For generative tasks (text completion, summarization):** - ```python - response = ModelResponse( - text=["The capital of France is Paris."], - input_tokens=[1, 2, 3, 4], - output_tokens=[[5, 6, 7, 8]] - ) - ``` - - **For multiple choice tasks:** - ```python - response = ModelResponse( - logprobs=[-0.5, -1.2, -2.1, -1.8], # Logprobs for each choice - argmax_logits_eq_gold=[False, False, False, False], # Whether correct choice was selected - input_tokens=[1, 2, 3, 4], - output_tokens=[[5], [6], [7], [8]] - ) - ``` - - **For perplexity calculation:** - ```python - response = ModelResponse( - text=["The model generated this text."], - logprobs=[-1.2, -0.8, -1.5, -0.9, -1.1], # Logprobs for each token - input_tokens=[1, 2, 3, 4, 5], - output_tokens=[[6], [7], [8], [9], [10]] - ) - ``` - - **For PMI analysis:** - ```python - response = ModelResponse( - text=["The answer is 42."], - logprobs=[-1.1, -0.9, -1.3, -0.7], # Conditioned logprobs - unconditioned_logprobs=[-2.1, -1.8, -2.3, -1.5], # Unconditioned logprobs - input_tokens=[1, 2, 3, 4], - output_tokens=[[5], [6], [7], [8]] - ) - ``` - - Notes: - - For most evaluation tasks, only a subset of attributes is required - - The `text` attribute is the most commonly used for generative tasks - - `logprobs` are essential for probability-based metrics like perplexity - """ - - # Model inputs - input: str | list | None = None - input_tokens: list[int] = field(default_factory=list) - - # Model text outputs - text: list[str] = field(default_factory=list) # The text of the response - output_tokens: list[list[int]] = field(default_factory=list) # Model generations - text_post_processed: list[str] | None = None # The text of the response postprocessed - reasonings: list[str | None] = field(default_factory=list) # The reasoning content of the response - - # Model logprob outputs - logprobs: list[float] = field(default_factory=list) # Log probabilities of the response - argmax_logits_eq_gold: list[bool] = field(default_factory=list) # Whether the argmax logits match the gold text - logits: list[list[float]] | None = None # Logits of the response, if applicable - unconditioned_logprobs: list[float] | None = None # Log probabilities of the unconditioned model (if applicable) - - # Other metadata - truncated_tokens_count: int = 0 # How many tokens truncated - padded_tokens_count: int = 0 # How many tokens of padding - - @property - def final_text(self) -> list[str]: - if self.text_post_processed is not None: - return self.text_post_processed - return self.text - - def __getitem__(self, index: int) -> "ModelResponse": - return ModelResponse( - input=self.input, - input_tokens=self.input_tokens, - text=[self.text[index]], - output_tokens=[self.output_tokens[index]] if self.output_tokens else [], - logprobs=[self.logprobs[index]] if self.logprobs else [], - argmax_logits_eq_gold=[self.argmax_logits_eq_gold[index]] if self.argmax_logits_eq_gold else [], - logits=[self.logits[index]] if self.logits else None, - unconditioned_logprobs=[self.unconditioned_logprobs[index]] if self.unconditioned_logprobs else None, - truncated_tokens_count=self.truncated_tokens_count, - padded_tokens_count=self.padded_tokens_count, - ) - - -@dataclass -class Batch: - input_ids: torch.Tensor - input_mask: torch.Tensor - input_lengths: list[int] - truncated: list[int] - padded: list[int] diff --git a/src/lighteval/models/nanotron/nanotron_model.py b/src/lighteval/models/nanotron/nanotron_model.py deleted file mode 100644 index 7ed6d35eb..000000000 --- a/src/lighteval/models/nanotron/nanotron_model.py +++ /dev/null @@ -1,1197 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# ruff: noqa: C901 -import logging -import os -import time -from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple, Type, Union - -import torch -import torch.nn.functional as F -import transformers -from datasets.download.streaming_download_manager import xPath -from pydantic import BaseModel -from torch.utils.data import DataLoader -from torch.utils.data.distributed import DistributedSampler -from tqdm import tqdm -from transformers import AutoTokenizer, BatchEncoding - -from lighteval.data import ( - GenDistributedSampler, - GenerativeTaskDatasetNanotron, - LoglikelihoodDataset, -) -from lighteval.models.model_output import ( - Batch, - ModelResponse, -) -from lighteval.models.transformers.transformers_model import LightevalModel -from lighteval.tasks.requests import ( - Doc, - SamplingMethod, -) -from lighteval.utils.cache_management import SampleCache, cached -from lighteval.utils.imports import is_package_available -from lighteval.utils.parallelism import find_executable_batch_size -from lighteval.utils.utils import as_list - - -logger = logging.getLogger(__name__) - - -os.environ["TOKENIZERS_PARALLELISM"] = "false" - -TokenSequence = Union[List[int], torch.LongTensor, torch.Tensor, BatchEncoding] - -if is_package_available("nanotron"): - from nanotron import distributed as dist - from nanotron import logging - from nanotron.config import GeneralArgs, ModelArgs, TokenizerArgs - from nanotron.config.parallelism_config import ParallelismArgs - from nanotron.generation.decode import decode_tokenized - from nanotron.generation.sampler import SamplerType - from nanotron.logging import human_format, log_rank - from nanotron.models import build_model - from nanotron.parallel.context import ParallelContext - from nanotron.parallel.parameters import sanity_check - from nanotron.parallel.pipeline_parallel.block import get_min_max_rank - from nanotron.parallel.pipeline_parallel.tensor_pointer import TensorPointer - from nanotron.parallel.tensor_parallel.enum import TensorParallelLinearMode - from nanotron.random import RandomStates, get_current_random_state, get_synced_random_state, set_random_seed - from nanotron.serialize import load_weights - from nanotron.trainer import CONFIG_TO_MODEL_CLASS, mark_tied_parameters - - logger = logging.get_logger(__name__) - -DEFAULT_GENERATION_SEED = 42 - - -class GenerationArgs(BaseModel): - sampler: Optional["SamplerType"] = None - temperature: Optional[float] = None - top_k: Optional[int] = None - top_p: Optional[float] = None - n_samples: Optional[int] = None - eos: Optional[str] = None - seed: Optional[int] = None - use_cache: Optional[bool] = False - - def __post_init__(self): - if self.seed is None: - self.seed = DEFAULT_GENERATION_SEED - - -@dataclass -class LightEvalLoggingArgs: - """Arguments related to logging for LightEval""" - - output_dir: str - results_path_template: str | None = None - save_details: bool = True - push_to_hub: bool = False - push_to_tensorboard: bool = False - public_run: bool = False - results_org: str | None = None - tensorboard_metric_prefix: str = "eval" - - -@dataclass -class LightEvalTasksArgs: - """Arguments related to tasks for LightEval""" - - tasks: str - custom_tasks: Optional[str] = None - max_samples: Optional[int] = None - num_fewshot_seeds: Optional[int] = None - - dataset_loading_processes: int = 8 - multichoice_continuations_start_space: Optional[bool] = None - pairwise_tokenization: bool = False - - -@dataclass -class LightEvalConfig: - """Arguments related to running LightEval on checkpoints. - - All is optional because you can also use this class to later supply arguments to override - the saved config when running LightEval after training. - """ - - logging: LightEvalLoggingArgs - tasks: LightEvalTasksArgs - parallelism: "ParallelismArgs" - batch_size: int = 0 - generation: Optional[Union[GenerationArgs, Dict[str, GenerationArgs]]] = None - - -@dataclass -class FullNanotronConfig: - lighteval_config: LightEvalConfig - nanotron_model: "ModelArgs" - nanotron_tokenizer: "TokenizerArgs" - nanotron_general: "GeneralArgs" - - @property - def generation_parameters(self): - # Return the generation parameters from the lighteval config - # or create default generation parameters if none are set - if self.lighteval_config.generation: - return self.lighteval_config.generation - return GenerationArgs() - - -class NanotronLightevalModel(LightevalModel): - # Default max sequence length setting for when no `max_length` is provided - # or no max length config setting is found in the model or tokenizer. - _DEFAULT_MAX_LENGTH: int = 2048 - - def __init__( - self, - checkpoint_path: str, - nanotron_config: FullNanotronConfig, - parallel_context: "ParallelContext", - max_gen_toks: Optional[int] = 256, - max_length: Optional[int] = None, - add_special_tokens: Optional[bool] = False, - dtype: Optional[Union[str, torch.dtype]] = None, - trust_remote_code: bool = False, - debug_one_layer_model: bool = False, - model_class: Optional[Type] = None, - ): - """Initializes a nanotron model for evaluation. - - Args: - checkpoint_path (str): Path to the model checkpoint. - nanotron_config (FullNanotronConfig): Configuration for the nanotron model. - parallel_context (ParallelContext): Parallel context for distributed training. - max_gen_toks (Optional[int]): Maximum number of tokens to generate. - max_length (Optional[int]): Maximum sequence length. - add_special_tokens (Optional[bool]): Whether to add special tokens. - dtype (Optional[Union[str, torch.dtype]]): Data type for the model. - trust_remote_code (bool): Whether to trust remote code. - debug_one_layer_model (bool): Whether to use a single layer for debugging. - model_class (Optional[Type]): Custom model class to use. - """ - self.config = nanotron_config - model_args = nanotron_config.nanotron_model - tokenizer = nanotron_config.nanotron_tokenizer - lighteval_config = nanotron_config.lighteval_config - parallel_config = nanotron_config.lighteval_config.parallelism - - self._max_gen_toks = max_gen_toks - self._max_length = max_length - self.parallel_config = parallel_config - self.parallel_context = parallel_context - - if parallel_config.pp > 1: - # To implement PP parallelism we need to think about how we want to sync the output for the PP ranks without outputs - raise ValueError("PP parallelism is not supported yet") - - # multichoice_continuations_start_space can be True (forcing space), False (forcing no space) or None (no forcing) - multichoice_continuations_start_space = lighteval_config.tasks.multichoice_continuations_start_space - - self.generation_config = lighteval_config.generation - if isinstance(self.generation_config, dict): - raise ValueError("We don't support yet generation configs per tasks") - - if dtype is None: - dtype = torch.bfloat16 - self.dtype = dtype - - self.model_config = model_args.model_config - if debug_one_layer_model: - self.model_config.num_hidden_layers = 1 - - self._add_special_tokens = add_special_tokens - self._tokenizer = self._create_auto_tokenizer( - pretrained=tokenizer.tokenizer_name_or_path, - trust_remote_code=trust_remote_code, - ) - self._tokenizer.model_max_length = self.max_length - - model_config_cls = self.model_config.__class__.__name__ - if model_class is not None: - CONFIG_TO_MODEL_CLASS[model_config_cls] = model_class - if model_config_cls not in CONFIG_TO_MODEL_CLASS: - raise ValueError( - f"Unsupported model config {model_config_cls}. Only {CONFIG_TO_MODEL_CLASS.keys()} are supported" - ) - - log_rank( - "Building model", - logger=logger, - level=logging.WARNING, - rank=0, - ) - - # Set random states - set_random_seed(42) - - # Get synchronized random states - if parallel_config.tp_mode is TensorParallelLinearMode.ALL_REDUCE: - random_states = RandomStates( - { - "tp_synced": get_synced_random_state( - random_state=get_current_random_state(), pg=parallel_context.tp_pg - ) - } - ) - else: - # We don't need to sync across TP when using sequence parallel (REDUCE_SCATTER) - random_states = RandomStates({}) - - model = build_model( - model_builder=lambda: CONFIG_TO_MODEL_CLASS[model_config_cls]( - config=self.model_config, - parallel_context=parallel_context, - parallel_config=parallel_config, - random_states=random_states, - ), - dtype=dtype, - parallel_context=parallel_context, - ) - - # Mark some parameters as tied - # TODO @nouamane: this is only needed for training, can we just mark params as NanotronParameter instead? - mark_tied_parameters(model=model, parallel_context=parallel_context, parallel_config=parallel_config) - - log_rank( - "Sanity checks on model", - logger=logger, - level=logging.WARNING, - rank=0, - ) - # Sanity check model - sanity_check(root_module=model) - - # Load checkpoint - log_rank( - f"Loading checkpoint from {checkpoint_path}:", - logger=logger, - level=logging.WARNING, - rank=0, - ) - load_weights(model=model, parallel_context=parallel_context, root_folder=xPath(checkpoint_path)) - model.eval() - - # We don't need the loss - self.model = model.transformer if model_config_cls == "FalconConfig" else model.model - - # Get the input and output ranks for the model (when using PP parallelism) - self.input_pp_rank, self.output_pp_rank = get_min_max_rank(module=self.model) - - self.multichoice_continuations_start_space = multichoice_continuations_start_space - self.pairwise_tokenization = nanotron_config.lighteval_config.tasks.pairwise_tokenization - self.batch_size = nanotron_config.lighteval_config.batch_size - - # Initialize cache for tokenization and predictions - self._cache = SampleCache(nanotron_config) - - @property - def tokenizer(self): - return self._tokenizer - - def _create_auto_tokenizer( - self, - *, - pretrained: str, - tokenizer: Optional[str] = None, - trust_remote_code: bool = False, - ) -> transformers.PreTrainedTokenizer: - """Returns a pre-trained tokenizer from a pre-trained tokenizer configuration.""" - try: - tokenizer = AutoTokenizer.from_pretrained( - pretrained if tokenizer is None else tokenizer, - trust_remote_code=trust_remote_code, - ) - except RecursionError: - tokenizer = AutoTokenizer.from_pretrained( - pretrained if tokenizer is None else tokenizer, - unk_token="", - trust_remote_code=trust_remote_code, - ) - tokenizer.pad_token = tokenizer.eos_token - tokenizer.padding_side = "left" - - return tokenizer - - @property - def add_special_tokens(self) -> bool: - """Whether to include special tokens in encoded text. This should be - determined by whether or not the model was trained with special tokens. - TODO: Remove these conditionals once HuggingFace supports a way to - check whether or not an arbitrary model was trained with special tokens. - """ - if self._add_special_tokens is not None: - return self._add_special_tokens - else: - return False - - @property - def eot_token(self) -> str: - return self.tokenizer.eos_token - - @property - def eot_token_id(self) -> int: - return self.tokenizer.eos_token_id - - @property - def max_gen_toks(self) -> int: - return self._max_gen_toks - - @property - def max_length(self) -> int: - """Return the maximum sequence length of the model. - NOTE: Different model configurations have different max sequence length - attribute names. - - n_positions: (CTRLConfig) - - max_position_embeddings: (BartConfig, RoFormerConfig) - - n_ctx: (GPT2Config) - NOTE: For relative position encoded models you should specify the max - sequence length of the model in the constructor via `max_length`. - """ - if self._max_length is not None: - return self._max_length - # Try to get the sequence length from the model config. - seqlen_config_attrs = ("n_positions", "max_position_embeddings", "n_ctx") - for attr in seqlen_config_attrs: - if hasattr(self.model_config, attr): - return getattr(self.model_config, attr) - if hasattr(self.tokenizer, "model_max_length"): - return self.tokenizer.model_max_length - return self._DEFAULT_MAX_LENGTH - - @property - def device(self) -> Union[int, str, torch.device]: - return "cuda" - - def _get_batch_size(self, max_input_length: int, override_bs: int = 0, starting_batch_size: int = 512) -> int: - if override_bs: - return override_bs - logger.warning("Detecting largest batch size") - - @find_executable_batch_size( - starting_batch_size=starting_batch_size - ) # if OOM, then halves batch_size and tries again - def forward_batch(batch_size): - logger.warning(f"Testing batch size {batch_size}") - test_batch = torch.ones( - (batch_size + int(0.1 * batch_size), max_input_length), device=self.device - ).long() # We add 10% for marging :) - F.log_softmax(self._model_call(test_batch).float(), dim=-1).cpu() - return batch_size - - batch_size = forward_batch() - logger.warning("Determined largest batch size: %d", batch_size) - return batch_size - - def tok_encode(self, string: str, add_special_tokens: Optional[bool] = None) -> TokenSequence: - # TODO: Merge `tok_encode_batch` here. - if add_special_tokens is None: - add_special_tokens = self.add_special_tokens - return self.tokenizer.encode(string, add_special_tokens=add_special_tokens) - - def tok_encode_batch(self, strings: List[str]) -> TokenSequence: - return self.tokenizer( - strings, - padding=True, - add_special_tokens=self.add_special_tokens, - return_tensors="pt", - ) - - def tok_decode(self, tokens: torch.LongTensor) -> List[str]: - return self.tokenizer.batch_decode(tokens, skip_special_tokens=True) - - def _model_call(self, inputs: torch.Tensor) -> torch.Tensor: - position_ids = ( - torch.arange(inputs.shape[1], device=inputs.device, dtype=torch.int32) - .unsqueeze(0) - .repeat(inputs.shape[0], 1) - ) - return self.model(inputs, position_ids) - - def homogeneize_ending_conditions(self, ending_condition: tuple | dict | list | str) -> tuple[list, int]: - """Ending conditions are submitted in several possible formats. - By default in lighteval we pass them as tuples (stop sequence, max number of items). - In the harness they sometimes are passed as dicts {"until": .., "max_length": ...} or - as only ending conditions, either lists or strings. - Here, we convert all these formats to a tuple containing a list of ending conditions, - and a float for the max length allowed. - """ - max_tokens, stop_sequences = None, None - # Filling with input values or default - if isinstance(ending_condition, tuple) and len(ending_condition) == 2: - stop_sequence_arg, max_gen_tokens_arg = ending_condition - stop_sequences = as_list(stop_sequence_arg) - max_tokens = max_gen_tokens_arg - elif isinstance(ending_condition, dict): # Tasks in the harness sometimes pass a dict to rf.greedy_until - try: - stop_sequences = as_list(ending_condition["until"]) - except KeyError: - stop_sequences = [] - try: - max_tokens = ending_condition["max_length"] - except KeyError: - max_tokens = self._max_gen_toks - else: # only gave stop sequences as an ending condition - stop_sequences = as_list(ending_condition) - - # Managing empty cases - if max_tokens is None: - max_tokens = self._max_gen_toks - if stop_sequences is None or (len(stop_sequences) == 1 and stop_sequences[0] is None): # or num_fewshot == 0: - stop_tokens = [self.eot_token] - else: - stop_tokens = list(stop_sequences) + [self.eot_token] - - assert isinstance(max_tokens, int) - assert isinstance(stop_tokens, list) - - return stop_tokens, max_tokens - - def _check_continuations_start_space(self, continuation: str) -> str: - """Some models tokenizer want a space at the beginning and other not. We update this if needed here. - multichoice_continuations_start_space can be: - - True (add a space if these isn't one) - - False (remove a space if there is one) - - None (Don't touch - default) - Todo: find a way to add this back WITHOUT breaking compatibility with the harness - """ - if self.multichoice_continuations_start_space is not None: - if self.multichoice_continuations_start_space and continuation[0] != " ": - continuation = " " + continuation - if not self.multichoice_continuations_start_space and continuation[0] == " ": - continuation = continuation.lstrip() - return continuation - - @cached(SamplingMethod.LOGPROBS) - def loglikelihood(self, requests: List[Doc]) -> List[ModelResponse]: - """Tokenize the context and continuation and compute the log likelihood of those - tokenized sequences. - """ - # requests = requests[:10256] - for request in tqdm( - requests, desc="Tokenizing", disable=bool(dist.get_rank(self.parallel_context.world_pg) != 0) - ): - if request.context == "": - request.tokenized_context = [self.tokenizer.eos_token_id] - request.tokenized_continuation = self.tok_encode(request.choice) - else: - # The following line is mandatory for compatibility with the harness - request.tokenized_context, request.tokenized_continuation = self.tok_encode_pair( - request.context, request.choice, self.pairwise_tokenization - ) - - return self._loglikelihood_tokens( - requests, - disable_tqdm=bool(dist.get_rank(self.parallel_context.world_pg) != 0), - ) - - @cached(SamplingMethod.PERPLEXITY) - def loglikelihood_rolling(self, requests: List[Doc]) -> List[ModelResponse]: - """This function is used to compute the log likelihood of the context for perplexity metrics.""" - for request in tqdm( - requests, desc="Tokenizing", disable=bool(dist.get_rank(self.parallel_context.world_pg) != 0) - ): # tuple of one elem - request.tokenized_context = [self.tokenizer.eos_token_id] # Fake context - request.tokenized_continuation = self.tok_encode(request.context) - - results = self._loglikelihood_tokens( - requests, - disable_tqdm=bool(dist.get_rank(self.parallel_context.world_pg) != 0), - return_bool_score=False, - ) - return results - - def prepare_batch( - self, - batch: List[str], - padding_length: int, - max_context: Optional[int] = None, - full_attention_masks: bool = False, - pad_on_left: bool = False, - ) -> Batch: - """Tokenize a batch of inputs and return also the length, truncations and padding - - We truncate to keep only at most `max_context` tokens - We pad to `padding_length` tokens - """ - assert full_attention_masks is False, ( - "full_attention_masks=True means we would be doing attention of padding tokens, which would affect negatively the results." - ) - assert pad_on_left is False, "pad_on_left=True not supported yet, see TODOs below" - current_pp_rank = dist.get_rank(self.parallel_context.pp_pg) - - if current_pp_rank != self.input_pp_rank: - # Nothing to do we're not on input GPUs - return TensorPointer(self.input_pp_rank), TensorPointer(self.input_pp_rank), -1, -1, -1 - - inputs = [] - attention_masks = [] - input_lengths = [] - truncated = [] - padded = [] - - if max_context is None: - max_context = self.max_length - - assert self.parallel_config.tp_mode == TensorParallelLinearMode.ALL_REDUCE, ( - "No reason to have tp_mode==REDUCE_SCATTER when doing inference" - ) - # if max_context % self.parallel_config.tp != 0: - # # We need to round up to the next multiple of self.parallel_config.tp - # if (max_context + (self.parallel_config.tp - max_context % self.parallel_config.tp)) < self.max_length: - # # We can add some tokens - # max_context = max_context + (self.parallel_config.tp - max_context % self.parallel_config.tp) - # else: - # # We need to remove some tokens - # max_context = max_context - (max_context % self.parallel_config.tp) - - # if padding_length % self.parallel_config.tp != 0: - # # We need to round up to the next multiple of self.parallel_config.tp - # if ( - # padding_length + (self.parallel_config.tp - padding_length % self.parallel_config.tp) - # ) < self.max_length: - # # We can add some tokens - # padding_length = padding_length + (self.parallel_config.tp - padding_length % self.parallel_config.tp) - # else: - # # We need to remove some tokens - # padding_length = padding_length - (padding_length % self.parallel_config.tp) - - # because vectorizing is annoying, we first convert each (context, continuation) pair to padded - # tensors, then we pack them together into a batch, call the model, and then pick it all apart - # again because vectorizing is annoying - - # Each sample is concatenated and cut to length or padded to max_length - for tokens in batch: - truncated.append(max(len(tokens) - max_context, 0)) - - # how this all works: - # CTX CONT - # inp 0 1 2 3|4 5 6 7 8 9 <- last token is deleted by inp[:, :-1] - # gpt2 \ \ - # logits 1 2 3|4 5 6 7 8 9 <- the ctx half gets tossed out by the - # cont_toks 4 5 6 7 8 9 [:, -len(continuation_enc):, :self.vocab_size] slice - - # when too long to fit in context, truncate from the left - inp = torch.tensor( - (tokens)[-max_context:], # [:-1], - dtype=torch.long, - ) - - (inplen,) = inp.shape - - # since in _collate we make sure length is descending, the longest is always the first one. - padding_length = padding_length if padding_length is not None else inplen - if padding_length - inplen < 0: - raise ValueError("Negative padding") - padded.append(padding_length - inplen) - - # pad length from seq to padding_length - if full_attention_masks: - att = torch.tensor([1] * padding_length, dtype=torch.bool) - else: - if pad_on_left: - att = torch.tensor([0] * (padding_length - inplen) + [1] * len(inp), dtype=torch.bool) - else: - att = torch.tensor([1] * len(inp) + [0] * (padding_length - inplen), dtype=torch.bool) - if pad_on_left: - inp = torch.cat( - [ - torch.zeros( - padding_length - inplen, dtype=torch.long - ), # [padding_length - seq] #TODO: padding_token not always 0 - inp, # [seq] - ], - dim=0, - ) - else: - inp = torch.cat( - [ - inp, # [seq] - torch.zeros( - padding_length - inplen, dtype=torch.long - ), # [padding_length - seq] #TODO: padding_token not always 0 - ], - dim=0, - ) - attention_masks.append(att.unsqueeze(0)) - - inputs.append(inp.unsqueeze(0)) # [1, padding_length] - input_lengths.append(inplen) - - input_ids = torch.cat(inputs, dim=0).to("cuda") # [batch, padding_length] - input_mask = torch.cat(attention_masks, dim=0).to("cuda") - - return Batch( - input_ids=input_ids, input_mask=input_mask, input_lengths=input_lengths, truncated=truncated, padded=padded - ) - - def gather(self, output_tensor: torch.Tensor, process_group: "dist.ProcessGroup" = None) -> torch.Tensor: - """Gather together tensors of (possibly) various size spread on separate GPUs (first exchange the lengths and then pad and gather)""" - if process_group is None: - process_group = self.parallel_context.dp_pg - output_tensor = output_tensor.contiguous() - gathered_outputs = [torch.zeros_like(output_tensor) for _ in range(process_group.size())] - dist.all_gather(gathered_outputs, output_tensor, group=process_group, async_op=False) - gathered_outputs = torch.cat(gathered_outputs) - return gathered_outputs - - def pad_and_gather(self, output_tensor: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]: - """Gather together tensors of (possibly) various size spread on separate GPUs (first exchange the lengths and then pad and gather)""" - if output_tensor.ndim == 1: - length_tensor = torch.tensor([output_tensor.shape[0]], device=self.device) - elif output_tensor.ndim == 2: - length_tensor = torch.tensor([output_tensor.shape[1]] * output_tensor.shape[0], device=self.device) - else: - raise ValueError("Unsupported tensor shape") - gathered_length = self.gather(length_tensor) - max_length = gathered_length.max().item() - - if output_tensor.ndim == 1: - output_tensor = F.pad( - output_tensor, (0, max_length - output_tensor.shape[0]), value=self.tokenizer.pad_token_id - ) - else: - output_tensor = F.pad( - output_tensor, (0, max_length - output_tensor.shape[1], 0, 0), value=self.tokenizer.pad_token_id - ) - gathered_outputs = self.gather(output_tensor) - - return gathered_outputs, gathered_length - - def _get_subsets(self, dataset, num_dataset_splits): - total_length = len(dataset) - subset_length = int(float(total_length) / float(num_dataset_splits)) + 1 - if subset_length < self.parallel_context.dp_pg.size(): - # We need at least one subset sample per DP process - subset_length = self.parallel_context.dp_pg.size() - return total_length, subset_length - - @torch.inference_mode() - def _loglikelihood_tokens( - self, - requests, - disable_tqdm: bool = False, - num_dataset_splits: int = 1, - return_bool_score: bool = True, - ) -> List[ModelResponse]: - dataset = LoglikelihoodDataset(requests=requests, num_dataset_splits=num_dataset_splits) - res = [] - - # Dataset is sorted in descending size. - # every 20-25% of the dataset we try to double the batch size for speed up - starting_batch_size = 512 - - total_length, subset_length = self._get_subsets(dataset, num_dataset_splits) - - for s, subset_start in enumerate( - tqdm( - range(0, total_length, subset_length), - disable=disable_tqdm, - position=0, - desc=f"loglikelihood -- Node {dist.get_rank(self.parallel_context.world_pg)}", - ) - ): - dataset.split_start = subset_start - dataset.split_end = min(subset_start + subset_length, total_length) - - # automatic (variable) batch size detection for vectorization - # pull longest context sample from request - context_enc = dataset[0].tokenized_context - continuation_enc = dataset[0].tokenized_continuation - - max_context = len((context_enc + continuation_enc)[-(self.max_length + 1) :][:-1]) - - batch_size = self._get_batch_size( - max_input_length=max_context, - override_bs=self.batch_size, - starting_batch_size=starting_batch_size, - ) - starting_batch_size = batch_size * 2 # for the next round - - # For the DP replicas - distributed_sampler = DistributedSampler( - dataset, - num_replicas=self.parallel_context.dp_pg.size(), - rank=dist.get_rank(self.parallel_context.dp_pg), - shuffle=False, - drop_last=False, - ) - to_remove_at_the_end = distributed_sampler.total_size - len(dataset) - - dataloader = DataLoader( - dataset, - batch_size=batch_size, - sampler=distributed_sampler, - drop_last=False, - collate_fn=lambda batch: batch, - ) - - tq = tqdm( - dataloader, - disable=disable_tqdm, - desc=f"loglikelihood in subset {s} Node {dist.get_rank(self.parallel_context.world_pg)}", - ) - - for j, batch_data in enumerate(tq): - if j < 3: - log_rank( - f"Memory usage: {torch.cuda.memory_allocated() / 1024**2:.2f}MB. Peak reserved memory: {torch.cuda.max_memory_reserved() / 1024**2:.2f}MB", - logger=logger, - level=logging.INFO, - group=self.parallel_context.world_pg, - rank=0, - ) - iteration_start_time = time.time() - inputs = [ - item.tokenized_context + item.tokenized_continuation[:-1] for item in batch_data - ] # The last token doesn't need to be input in the model - - pad_on_left = ( - False # if we unpad in modeling, it doesn't matter if left or right # TODO: not supported yet - ) - batch_model = self.prepare_batch( - inputs, - padding_length=max_context, - max_context=max_context, - full_attention_masks=False, - pad_on_left=pad_on_left, - ) - # batched_inputs, batch_attention, input_lengths, truncated, padded - with torch.no_grad(): - # Create sequential position_ids initialized to -1 (for padding) - position_ids = torch.full( - batch_model.input_ids.shape, fill_value=-1, dtype=torch.long, device=self.device - ) - - for i, length in enumerate(batch_model.input_lengths): - if pad_on_left: - position_ids[i, -length:] = torch.arange(length, device=self.device, dtype=torch.int32) - else: - position_ids[i, :length] = torch.arange(length, device=self.device, dtype=torch.int32) - out = self.model(input_ids=batch_model.input_ids, position_ids=position_ids) - - if dist.get_rank(self.parallel_context.pp_pg) == self.output_pp_rank: - # This process got outputs - - # Gather all the output across TP - gathered_out = [torch.zeros_like(out) for _ in range(self.parallel_context.tp_pg.size())] - dist.all_gather(gathered_out, out, group=self.parallel_context.tp_pg, async_op=False) - out = torch.cat(gathered_out, dim=-1) - - out = out.view(*batch_model.input_ids.shape, -1) # [batch, seq_length, vocab] - multi_logits = F.log_softmax(out, dim=-1) # [batch, padding_length, vocab] - - logits_sum = [] - max_equals = [] - batch_cont_tokens = [] - for cur_request, cur_logits, inplen in zip(batch_data, multi_logits, batch_model.input_lengths): - cont_toks = torch.tensor( - cur_request.tokenized_continuation, dtype=torch.long, device=self.device - ) - contlen = cont_toks.shape[0] - # We only look at the continuation tokens - if contlen > inplen: - # Continuation is longer than the input size, we are in rolling mode (only continuation) - cur_logits = cur_logits.unsqueeze(0).to(self.device) # [1, seq, vocab] - cont_toks = cont_toks[:inplen].unsqueeze(0).to(self.device) # [1, seq] - else: - # if contlen == 1: - # top_k = torch.topk(logits, 20)[1].tolist() - # if any(bool(el not in top_k) for el in cont_toks) and not printed_error: - # top_toks_str = '|'.join(self.tokenizer.decode(tt).replace('\n', '') for tt in top_k) - # cont_toks_str = '|'.join(self.tokenizer.decode(tt).replace('\n', '') for tt in cont_toks.tolist()) - # logger.error( - # f"Not all the solutions are in the top 20 most likely tokens on rank {dist.get_rank(self.parallel_context.world_pg)} " - # f"top_tokens: {top_toks_str}\ncont_tokens: {cont_toks_str}") - - cur_logits = ( - cur_logits[inplen - contlen : inplen].unsqueeze(0).to(self.device) - ) # [1, seq, voc] #TODO: this assumes padding on the right - cont_toks = cont_toks.unsqueeze(0).to(self.device) # [1, seq] - - # Check if per-token argmax is exactly equal to continuation - greedy_tokens = cur_logits.argmax(dim=-1).to(self.device) - # Sometimes the continuation is longer than allowed by the model, we only look at the first tokens - max_equal = (greedy_tokens == cont_toks).all().squeeze(0).to(self.device) - - # Obtain log-probs at the corresponding continuation token indices - cur_logits = torch.gather(cur_logits, 2, cont_toks.unsqueeze(-1)).squeeze(-1) # [1, seq] - - # Answer: (log prob, is-exact-match) - logits_sum.append(cur_logits.sum()) - max_equals.append(max_equal) - batch_cont_tokens.append(cont_toks) - - # Sync all - # Need reshaping before gather - batched_inputs, _ = self.pad_and_gather(batch_model.input_ids) - lengths = torch.tensor(batch_model.input_lengths, device=self.device) - batched_lengths = self.gather(lengths) - cont_lengths = torch.tensor([len(c[0]) for c in batch_cont_tokens], device=self.device) - batch_cont_length = self.gather(cont_lengths) - max_cont_tokens_length = max(len(c[0]) for c in batch_cont_tokens) - batch_cont_tokens = torch.cat( - [ - F.pad(c, (0, max_cont_tokens_length - c.shape[1], 0, 0), value=self.tokenizer.pad_token_id) - for c in batch_cont_tokens - ], - dim=0, - ) - batch_cont_tokens, _ = self.pad_and_gather(batch_cont_tokens) - # Can be gathered as such - logits = torch.tensor(logits_sum, device=self.device) - logits = self.gather(logits) - max_equal = torch.tensor(max_equals, device=self.device) - max_equal = self.gather(max_equal) - batch_truncated = torch.tensor(batch_model.truncated, device=self.device) - batch_truncated = self.gather(batch_truncated) - batch_padded = torch.tensor(batch_model.padded, device=self.device) - batch_padded = self.gather(batch_padded) - - batch_res = [] - for ix, ( - logit, - cont_tokens, - cont_length, - maxe, - batched_input, - batched_length, - trunc, - padded, - ) in enumerate( - zip( - logits, - batch_cont_tokens, - batch_cont_length, - max_equal, - batched_inputs, - batched_lengths, - batch_truncated, - batch_padded, - ) - ): - answer = ModelResponse( - result=(float(logit), bool(maxe)) if return_bool_score else float(logit.sum()), - input_tokens=batched_input[: batched_length.item()].numpy(force=True), - generated_tokens=cont_tokens[: cont_length.item()].numpy(force=True), - truncated_tokens_count=trunc.cpu().item(), - padded_tokens_count=padded.cpu().item(), - ) - batch_res.append(answer) - - # Sort batches back when we add then in res - because of the DistributedSampler, they are interleaved in the results: - # Ex with DP=3 and a batch of size 3 we end up with A D G B E H C F I instead of A B C D E F G H I - assert len(batch_res) % self.parallel_context.dp_pg.size() == 0 - for i in range(len(batch_res) // self.parallel_context.dp_pg.size()): - for j in range(self.parallel_context.dp_pg.size()): - res.append(batch_res[i + j * (len(batch_res) // self.parallel_context.dp_pg.size())]) - - # A bit of logging - elapsed_time_per_iteration_ms = (time.time() - iteration_start_time) * 1000 - tokens_per_sec = batched_inputs.numel() / (elapsed_time_per_iteration_ms / 1000) - tq.desc = f"loglikelihood Subset {s} Node {dist.get_rank(self.parallel_context.world_pg)} - {human_format(tokens_per_sec)} tokens/s" - - # Clean up GPUs - del out - del logits - del batched_inputs - del batch_truncated - del batch_padded - - # At the end of the subset, remove the additional samples we may have added to make the dataset divisible by the number of processes - assert to_remove_at_the_end >= 0 - res = res[: len(res) - to_remove_at_the_end] - - # if dist.get_rank(self.parallel_context.tp_pg) == 0: - # for i, r in enumerate(res): - # print(f"i {i} results: {r.result[-30:]}") - # print(f"i {i} input_tokens: {r.input_tokens[-(r.padded + 10):-r.padded]}") - # print(f"i {i} cont_tokens: {r.cont_tokens[-30:]}") - # print(f"i {i} truncated: {r.truncated}") - # print(f"i {i} padded: {r.padded}") - - if dist.get_rank(self.parallel_context.pp_pg) == self.output_pp_rank: - assert len(res) == total_length, "we didn't cover all the data" - - if len(res) == 0: - # We are in a process which return no output (beginning/middle of the PP group) - return [] - - return dataset.get_original_order(res) - - @torch.inference_mode() - @cached(SamplingMethod.GENERATIVE) - def greedy_until( - self, - requests: List[Doc], - disable_tqdm: bool = False, - num_dataset_splits: int = 1, - ) -> List[ModelResponse]: - """Greedy generation until a stop token is generated.""" - # automatic (variable) batch size detection for vectorization - # pull longest context sample from request - for request in requests: - request.stop_sequence = as_list(request.stop_sequence) + [self.tokenizer.eos_token] - request.tokenized_context = self.tok_encode(request.context) - - dataset = GenerativeTaskDatasetNanotron(requests=requests, num_dataset_splits=num_dataset_splits) - res = [] - - # Dataset is sorted in descending size. - # every 20-25% of the dataset we try to double the batch size for speed up - starting_batch_size = 512 - - total_length, subset_length = self._get_subsets(dataset, num_dataset_splits) - - for s, subset_start in enumerate( - tqdm( - range(0, total_length, subset_length), - disable=disable_tqdm, - position=0, - desc=f"greedy -- Node {dist.get_rank(self.parallel_context.world_pg)}", - ) - ): - dataset.split_start = subset_start - dataset.split_end = min(subset_start + subset_length, total_length) - - if dataset[0][1].generation_size is None: - # No constraints on the generation size: max length allowed is the max model context - max_input_length = self.max_length - else: - # Longest context in the current split is the first item (since we sort reversed) - context_enc = dataset[0][1].tokenized_context - max_gen = max(item.generation_size for item in dataset) - max_input_length = min(len(context_enc) + max_gen, self.max_length) - - batch_size = self._get_batch_size( - override_bs=self.batch_size, - max_input_length=max_input_length, - starting_batch_size=starting_batch_size, - ) - # For next iteration, since the batch will be smaller, we'll test a bigger batch size - starting_batch_size = batch_size * 2 - - # For the DP replicas - distributed_sampler = GenDistributedSampler( - dataset, - num_replicas=self.parallel_context.dp_pg.size(), - rank=dist.get_rank(self.parallel_context.dp_pg), - shuffle=False, - drop_last=False, - ) - to_remove_at_the_end = distributed_sampler.total_size - len(dataset) - - dataloader = DataLoader( - dataset, - batch_size=batch_size, - sampler=distributed_sampler, - drop_last=False, - collate_fn=lambda batch: batch, - ) - - tq = tqdm(dataloader, desc=f"greedy in subset {s} Node {dist.get_rank(self.parallel_context.world_pg)}") - for j, indexed_batch in enumerate(tq): - if j < 3: - log_rank( - f"Memory usage: {torch.cuda.memory_allocated() / 1024**2:.2f}MB. Peak reserved memory: {torch.cuda.max_memory_reserved() / 1024**2:.2f}MB", - logger=logger, - level=logging.INFO, - group=self.parallel_context.world_pg, - rank=0, - ) - iteration_start_time = time.time() - sample_index, batch = zip(*indexed_batch) - - # NOTE: we are assuming all items in a batch behave similarly (same - # stop_tokens and max_tokens genrated) which is not necessarily - # the case! Because of that we should only use batch size of 1 - - # Since items are sorted by inverse length, the first one always has - # the maximum allowed generation size for the batch, unless we want to force truncation - # need to pass them somewhere ! stop_tokens = batch[0].stop_sequence - max_new_tokens = batch[0].generation_size - returns_logits = batch[0].use_logits - num_samples = batch[0].num_samples - if num_samples > 1: - logger.error( - "Nonotron models does not allow sampling evaluations - this is likely to fail or provide problematic results" - ) - - context = [c.context for c in batch] - - # See doc https://huggingface.co/docs/transformers/v4.38.2/en/pad_truncation#padding-and-truncation - # Will do left truncation and padding, as defined when creating the tokenizer - tokenized = self.tokenizer( - context, - truncation="longest_first", # we truncate to the model max length if needed - padding="longest", # we pad to the longest sequence - return_tensors="pt", - max_length=self.max_length - 1, # we always allow minimum one token of generation - add_special_tokens=self.add_special_tokens, - ).to(self.device) - - # The main question for this step is the following: - # Would we rather truncate the prompt to allow generation to go to max_new_tokens, at the risk - # of losing some meaning, or have some generations that are exceedingly short? - # The choice we go for here is to avoid truncating the prompt if we can, since it - # should have been managed by the prompt creator/few shot manager if requested by the user. - context_size = tokenized["input_ids"].shape[1] - if context_size > self.max_length: - logger.warning( - f"The context size of your batch ({context_size}) is bigger than the maximum context size allowed by the model ({self.max_length}) for a task in" - + str({i.task_name for i in batch}) - + ". This is likely to lead to some errors." # noqa C401 - ) - # There will be truncation of at least one sample, maximum generation size will be one - max_new_tokens = 1 - else: # We can't allow generation of more than max_length - max_new_tokens = min(self.max_length - context_size, max_new_tokens) - - batch_model = Batch( - input_ids=tokenized["input_ids"], - input_lengths=[len(item == 1) for item in tokenized["attention_mask"]], - input_mask=tokenized["attention_mask"], - truncated=[ - len(c) - tokenized["input_ids"].shape[1] if len(c) > tokenized["input_ids"].shape[1] else 0 - for c in context - ], - padded=[sum(mask == 0) for mask in tokenized["attention_mask"]], - ) - - # responses, logits and input_ids have all been gathered across GPUs already - # but we also grab the original length of these vectors, which have been padded - # while being gathered - the added info - outputs = decode_tokenized( - input_ids=batch_model.input_ids, - input_mask=batch_model.input_mask, - model=self.model, - parallel_context=self.parallel_context, - max_new_tokens=max_new_tokens, - max_micro_batch_size=batch_size, # ok for PP=1 for PP>1 we'll need to split the batch - returns_logits=returns_logits, - tokenizer=self.tokenizer, - generation_config=self.generation_config, - ) - dist.barrier() # Got everyone to send their stuff - outputs = list(outputs) - - generations = torch.stack([o.generation_ids[o.input_ids.shape[0] :] for o in outputs]) - batch_input_ids, len_ids = self.pad_and_gather(batch_model.input_ids) - batch_generations, _ = self.pad_and_gather(generations) - - if returns_logits: - logits = torch.stack([o.logits for o in outputs]) - logits, len_logits = self.pad_and_gather(logits) - - if dist.get_rank(self.parallel_context.pp_pg) == self.output_pp_rank: - generations = batch_generations.numpy(force=True) - input_ids = batch_input_ids.numpy(force=True) - - batch_sample_index = torch.tensor(sample_index, device=self.device) - batch_sample_index = self.gather(batch_sample_index) - batch_truncated = torch.tensor(batch_model.truncated, device=self.device) - batch_truncated = self.gather(batch_truncated) - batch_padded = torch.tensor(batch_model.padded, device=self.device) - batch_padded = self.gather(batch_padded) - - batch_res = [] - for ix, (generation, batched_input, trunc, padded, sample_index) in enumerate( - zip(generations, input_ids, batch_truncated, batch_padded, batch_sample_index) - ): - # Ensure the generated responses do not contain the stop sequences. - decoded_response = self.tokenizer.decode(generation, skip_special_tokens=False) - stop_terms = dataset[sample_index][1].stop_sequence - for stop_term in stop_terms: - decoded_response = decoded_response.split(stop_term)[0] - # partial caching - cur_response = ModelResponse( - result=decoded_response, - logits=logits[ix][: len_logits[ix]] if returns_logits else None, - generated_tokens=generation, - input_tokens=batched_input[: len_ids[ix]], - truncated_tokens_count=trunc.cpu().item(), - padded_tokens_count=padded.cpu().item(), - ) - # self.cache_hook.add_partial("greedy_until", (context, stop_tokens), cur_response) - batch_res.append(cur_response) - - # Sort batches back when we add then in res - because of the DistributedSampler, they are interleaved in the results: - # Ex with DP=3 and a batch of size 3 we end up with A D G B E H C F I instead of A B C D E F G H I - assert len(batch_res) % self.parallel_context.dp_pg.size() == 0 - for i in range(len(batch_res) // self.parallel_context.dp_pg.size()): - for j in range(self.parallel_context.dp_pg.size()): - res.append(batch_res[i + j * (len(batch_res) // self.parallel_context.dp_pg.size())]) - - # A bit of logging - elapsed_time_per_iteration_ms = (time.time() - iteration_start_time) * 1000 - tokens_per_sec = (batch_input_ids.numel() + batch_generations.numel()) / ( - elapsed_time_per_iteration_ms / 1000 - ) - - tq.desc = f"greedy_until Subset {s} Node {dist.get_rank(self.parallel_context.world_pg)} - {human_format(tokens_per_sec)} tokens/s" - - # At the end of the subset, remove the additional samples we may have added to make the dataset divisible by the number of processes - assert to_remove_at_the_end >= 0 - res = res[: len(res) - to_remove_at_the_end] - - if dist.get_rank(self.parallel_context.pp_pg) == self.output_pp_rank: - assert len(res) == total_length, ( - f"we didn't cover all the data: len(res) == total_length ({len(res)} == {total_length})" - ) - - if len(res) == 0: - # We are in a process which return no output (beginning/middle of the PP group) - return [] - - return dataset.get_original_order(res) - - -class MultiTokenEOSCriteria(transformers.StoppingCriteria): - """Criteria to stop on the specified multi-token sequence.""" - - def __init__( - self, - sequence: str, - tokenizer: transformers.PreTrainedTokenizer, - initial_decoder_input_length: int, - batch_size: int, - ): - self.initial_decoder_input_length = initial_decoder_input_length - self.done_tracker = [False] * batch_size - self.sequence = sequence - self.sequence_ids = tokenizer.encode(sequence, add_special_tokens=False) - self.sequence_id_len = len(self.sequence_ids) - self.tokenizer = tokenizer - - def __call__(self, input_ids, scores, **kwargs) -> bool: - # For efficiency, we compare the last n tokens where n is the number of tokens in the stop_sequence - lookback_ids_batch = input_ids[:, self.initial_decoder_input_length :][:, -self.sequence_id_len :] - - lookback_tokens_batch = self.tokenizer.batch_decode(lookback_ids_batch) - - for i, done in enumerate(self.done_tracker): - if not done: - self.done_tracker[i] = self.sequence in lookback_tokens_batch[i] - return False not in self.done_tracker diff --git a/src/lighteval/models/sglang/sglang_model.py b/src/lighteval/models/sglang/sglang_model.py deleted file mode 100644 index e5c0f4d87..000000000 --- a/src/lighteval/models/sglang/sglang_model.py +++ /dev/null @@ -1,422 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The SGLang Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import gc -import logging -from typing import Optional - -import torch -from pydantic import PositiveFloat, PositiveInt -from tqdm import tqdm - -from lighteval.data import GenerativeTaskDataset, LoglikelihoodDataset -from lighteval.models.abstract_model import LightevalModel, ModelConfig -from lighteval.models.model_output import ModelResponse -from lighteval.models.utils import _simplify_name, uses_chat_template -from lighteval.tasks.prompt_manager import PromptManager -from lighteval.tasks.requests import Doc, SamplingMethod -from lighteval.utils.cache_management import SampleCache, cached -from lighteval.utils.imports import is_package_available, requires - - -logger = logging.getLogger(__name__) - -if is_package_available("sglang"): - from sglang import Engine - from sglang.srt.hf_transformers_utils import get_tokenizer - - logging.getLogger("sglang").propagate = True - logging.getLogger("sglang").handlers.clear() -else: - Engine = None - get_tokenizer = None - - -class SGLangModelConfig(ModelConfig): - """Configuration class for SGLang inference engine. - - This configuration is used to load and configure models using the SGLang inference engine, - which provides high-performance inference. - - sglang doc: https://docs.sglang.ai/index.html# - - Attributes: - model_name (str): - HuggingFace Hub model ID or path to the model to load. - load_format (str): - The format of the model weights to load. choices: auto, pt, safetensors, npcache, dummy, tensorizer, sharded_state, gguf, bitsandbytes, mistral, runai_streamer. - dtype (str): - Data type for model weights. Defaults to "auto". Options: "auto", "float16", "bfloat16", "float32". - tp_size (PositiveInt): - Number of GPUs to use for tensor parallelism. Defaults to 1. - dp_size (PositiveInt): - Number of GPUs to use for data parallelism. Defaults to 1. - context_length (PositiveInt | None): - Maximum context length for the model. - random_seed (PositiveInt | None): - Random seed for reproducibility. Defaults to 1234. - trust_remote_code (bool): - Whether to trust remote code when loading models. Defaults to False. - device (str): - Device to load the model on. Defaults to "cuda". - skip_tokenizer_init (bool): - Whether to skip tokenizer initialization. Defaults to False. - kv_cache_dtype (str): - Data type for key-value cache. Defaults to "auto". - add_special_tokens (bool): - Whether to add special tokens during tokenization. Defaults to True. - pairwise_tokenization (bool): - Whether to tokenize context and continuation separately for loglikelihood evals. Defaults to False. - sampling_backend (str | None): - Sampling backend to use. If None, uses default. - attention_backend (str | None): - Attention backend to use. If None, uses default. - mem_fraction_static (PositiveFloat): - Fraction of GPU memory to use for static allocation. Defaults to 0.8. - chunked_prefill_size (PositiveInt): - Size of chunks for prefill operations. Defaults to 4096. - override_chat_template (bool): - If True, we force the model to use a chat template. If alse, we prevent the model from using - a chat template. If None, we use the default (true if present in the tokenizer, false otherwise) - generation_parameters (GenerationParameters, optional, defaults to empty GenerationParameters): - Configuration parameters that control text generation behavior, including - temperature, top_p, max_new_tokens, etc. - system_prompt (str | None, optional, defaults to None): Optional system prompt to be used with chat models. - This prompt sets the behavior and context for the model during evaluation. - cache_dir (str, optional, defaults to "~/.cache/huggingface/lighteval"): Directory to cache the model. - - Example: - ```python - config = SGLangModelConfig( - model_name="meta-llama/Llama-3.1-8B-Instruct", - tp_size=2, - context_length=8192, - generation_parameters=GenerationParameters( - temperature=0.7, - max_new_tokens=100 - ) - ) - ``` - """ - - model_name: str - load_format: str = "auto" - dtype: str = "auto" - tp_size: PositiveInt = 1 # how many GPUs to use for tensor parallelism - dp_size: PositiveInt = 1 # how many GPUs to use for data parallelism - context_length: PositiveInt | None = None - random_seed: PositiveInt | None = 1234 - trust_remote_code: bool = False - device: str = "cuda" - skip_tokenizer_init: bool = False - kv_cache_dtype: str = "auto" - add_special_tokens: bool = True - pairwise_tokenization: bool = False - sampling_backend: str | None = None - attention_backend: str | None = None - mem_fraction_static: PositiveFloat = 0.8 - chunked_prefill_size: PositiveInt = 4096 - override_chat_template: bool = None - - -class SGLangModel(LightevalModel): - def __init__( - self, - config: SGLangModelConfig, - ): - """Initializes an SGLang model.""" - self.config = config - self.use_chat_template = uses_chat_template( - model_name=self.config.model_name, override_chat_template=config.override_chat_template - ) - self.data_parallel_size = config.dp_size - self.tensor_parallel_size = config.tp_size - self._add_special_tokens = config.add_special_tokens - self._tokenizer = self._create_auto_tokenizer(config) - self._max_length = config.context_length if config.context_length is not None else None - self.model = self._create_auto_model(config) - self.model_name = _simplify_name(config.model_name) - self.model_sha = "" # config.get_model_sha() - self.precision = config.dtype - self.sampling_params = config.generation_parameters.to_sglang_dict() - self.sampling_backend = config.sampling_backend - self.attention_backend = config.attention_backend - self.pairwise_tokenization = config.pairwise_tokenization - self.prompt_manager = PromptManager(self.use_chat_template, self.tokenizer, config.system_prompt) - - # Initialize cache for tokenization and predictions - self._cache = SampleCache(config) - - @property - def tokenizer(self): - return self._tokenizer - - def cleanup(self): - if self.model is not None: - self.model.shutdown() - - self.model = None - gc.collect() - torch.cuda.empty_cache() - - @property - def add_special_tokens(self): - return self._add_special_tokens - - @property - def max_length(self) -> int: - return self._max_length - - def _create_auto_model(self, config: SGLangModelConfig) -> Optional["Engine"]: - self.model_args = { - "model_path": config.model_name, - "trust_remote_code": config.trust_remote_code, - "dtype": config.dtype, - "device": "cuda", - "random_seed": config.random_seed, - "load_format": config.load_format, - "context_length": self._max_length, - "dp_size": config.dp_size, - "tp_size": config.tp_size, - "sampling_backend": config.sampling_backend, - "attention_backend": config.attention_backend, - "mem_fraction_static": config.mem_fraction_static, - "schedule_policy": "fcfs", - "chunked_prefill_size": config.chunked_prefill_size, - "disable_radix_cache": True, - } - model = Engine(**self.model_args) - - if self._max_length is None: - self._max_length = 8192 - - return model - - def _create_auto_tokenizer(self, config: SGLangModelConfig): - tokenizer = get_tokenizer( - config.model_name, - tokenizer_mode="auto", - trust_remote_code=config.trust_remote_code, - tokenizer_revision="main", - ) - tokenizer.pad_token = tokenizer.eos_token - return tokenizer - - @cached(SamplingMethod.GENERATIVE) - def greedy_until( - self, - docs: list[Doc], - ) -> list[ModelResponse]: - """Generates responses using a greedy decoding strategy until certain ending conditions are met. - - Args: - docs (list[Doc]): List of documents containing the context for generation. - - Returns: - list[ModelResponse]: list of generated responses. - """ - return self._greedy_until(docs) - - def _greedy_until( - self, - docs: list[Doc], - ) -> list[ModelResponse]: - dataset = GenerativeTaskDataset(requests=docs, num_dataset_splits=self.DATASET_SPLITS) - results = [] - - for split in tqdm( - dataset.splits_iterator(), - total=dataset.num_dataset_splits, - desc="Splits", - position=0, - disable=False, - ): - if self.use_chat_template: - stop_tokens = [] - else: - stop_tokens = split[0].stop_sequences - - max_new_tokens = split[0].generation_size # could be none - num_samples = split[0].num_samples - - contexts = [self.prompt_manager.prepare_prompt(doc) for doc in split] - tokenized = self.tokenizer(contexts, add_special_tokens=self.add_special_tokens) - - # The main question for this step is the following: - # Would we rather truncate the prompt to allow generation to go to max_new_tokens, at the risk - # of losing some meaning, or have some generations that are exceedingly short? - # The choice we go for here is to avoid truncating the prompt if we can, since it - # should have been managed by the prompt creator/few shot manager if requested by the user. - - inputs = tokenized["input_ids"] - context_size = len(inputs[0]) - - # left truncate the inputs to the maximum length - if max_new_tokens is not None: - if context_size + max_new_tokens > self.max_length: - logger.warning( - f"{context_size + max_new_tokens=} which is greater than {self.max_length=}. Truncating context to {self.max_length - max_new_tokens} tokens." - ) - context_size = self.max_length - max_new_tokens - if context_size < 0: - logger.critical( - f"{context_size=} is less than 0, either reduce the max_new_tokens or increase model max length." - ) - raise ValueError("Context size is less than 0.") - inputs = [input[-context_size:] for input in inputs] - else: - if context_size > self.max_length: - logger.warning( - f"{context_size=} which is greater than {self.max_length=}. Truncating context to {self.max_length} tokens." - ) - context_size = self.max_length - inputs = [input[-context_size:] for input in inputs] - - sglang_outputs = self._generate( - inputs=inputs, - max_new_tokens=max_new_tokens, - stop_tokens=stop_tokens, - num_samples=num_samples, - ) - - for input_token_ids, sglang_output in zip(inputs, sglang_outputs): - meta_info = sglang_output["meta_info"] - output_token_logprobs = meta_info["output_token_logprobs"] - output_token_ids = [output[1] for output in output_token_logprobs] - logprobs = [output[0] for output in output_token_logprobs] - result = [sglang_output["text"]] - cur_response = ModelResponse( - text=result, - logprobs=logprobs, - output_tokens=list(output_token_ids), - input_tokens=input_token_ids, - ) - results.append(cur_response) - return dataset.get_original_order(results) - - @requires("sglang") - def _generate( - self, - inputs: list[list[int]], - max_new_tokens: Optional[int] = None, - stop_tokens: Optional[list[str]] = None, - num_samples: int = 1, - generate: bool = True, - ) -> list: - """Contains the actual logic of the generation.""" - logprob_start_len = None - top_logprobs_num = None - if generate: - self.sampling_params["max_new_tokens"] = max_new_tokens - self.sampling_params["stop"] = stop_tokens - self.sampling_params["n"] = num_samples - if num_samples > 1 and self.sampling_params["temperature"] == 0: - raise ValueError( - "num_samples > 1 is not supported with temperature=0, please set temperature > 0 or use non sampling metrics." - ) - else: - self.sampling_params["max_new_tokens"] = 1 - self.sampling_params["temperature"] = 0 - logprob_start_len = 0 - top_logprobs_num = 1 - - outputs = self.model.generate( - input_ids=inputs, - sampling_params=self.sampling_params, - return_logprob=True, - logprob_start_len=logprob_start_len, - top_logprobs_num=top_logprobs_num, - ) - return outputs - - @cached(SamplingMethod.LOGPROBS) - def loglikelihood(self, docs: list[Doc]) -> list[ModelResponse]: - return self._loglikelihood_tokens(docs) - - def _loglikelihood_tokens( - self, - docs: list[Doc], - ) -> list[ModelResponse]: - dataset = LoglikelihoodDataset(requests=docs, num_dataset_splits=1) - res = [] - - for split in tqdm(dataset.splits_iterator(), disable=False): - contexts = [self.prompt_manager.prepare_prompt(doc) for doc in split] - # the last token is an eos token, so we don't need to add it - # Left truncate the inputs to the maximum length - inputs = [] - tokenized_continuations_batch = [] - tokenized_contexts_batch = [] - - for context, doc in zip(contexts, dataset): - tokenized_contexts, tokenized_continuations = self.tok_encode_pair(context, doc.choices, pairwise=True) - for tokenized_context, tokenized_continuation in zip(tokenized_contexts, tokenized_continuations): - inputs.append(tokenized_context + tokenized_continuation) - tokenized_continuations_batch.append(tokenized_continuation) - tokenized_contexts_batch.append(tokenized_context) - - inputs = [input[-self.max_length :] for input in inputs] - outputs = self._generate(inputs, generate=False) - - flat_index = 0 - for doc in dataset: - # all the element generated from one doc (one element per choice) - outputs_doc: list[dict] = outputs[flat_index : flat_index + len(doc.choices)] - tokenized_continuations_doc: list[list[int]] = tokenized_continuations_batch[ - flat_index : flat_index + len(doc.choices) - ] - tokenized_contexts_doc: list[list[int]] = tokenized_contexts_batch[ - flat_index : flat_index + len(doc.choices) - ] - logprobs_doc = [] - argmax_doc = [] - output_tokens_doc = [] - input_tokens_doc = [] - - for output, context, continuation in zip( - outputs_doc, tokenized_contexts_doc, tokenized_continuations_doc - ): - meta_info = output["meta_info"] - - input_top_logprobs = meta_info["input_top_logprobs"][::-1] - input_token_logprobs = meta_info["input_token_logprobs"][::-1] - input_top_logprobs = input_top_logprobs[: len(continuation)] - logprobs = input_token_logprobs[: len(continuation)] - bool_score = all(top[0][1] == input[1] for top, input in zip(input_top_logprobs, logprobs)) - logprobs = [logprob[0] for logprob in logprobs] - logprobs_doc.append(logprobs) - argmax_doc.append(bool_score) - output_tokens_doc.append(output["text"]) - input_tokens_doc.append(context + continuation) - - answer = ModelResponse( - input_tokens=input_tokens_doc, - output_tokens=output_tokens_doc, - logprobs=logprobs_doc, - argmax_logits_eq_gold=argmax_doc, - ) - res.append(answer) - return dataset.get_original_order(res) - - @cached(SamplingMethod.PERPLEXITY) - def loglikelihood_rolling(self, docs: list[Doc]) -> list[ModelResponse]: - raise NotImplementedError() diff --git a/src/lighteval/models/transformers/adapter_model.py b/src/lighteval/models/transformers/adapter_model.py deleted file mode 100644 index 52f339664..000000000 --- a/src/lighteval/models/transformers/adapter_model.py +++ /dev/null @@ -1,127 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import logging -import shutil -from contextlib import nullcontext - -import torch -import transformers -from transformers import AutoModelForCausalLM - -from lighteval.models.transformers.transformers_model import TransformersModel, TransformersModelConfig -from lighteval.models.utils import _get_dtype -from lighteval.utils.imports import is_package_available, requires - - -logger = logging.getLogger(__name__) - -if is_package_available("peft"): - from peft import PeftModel - - -@requires("peft") -class AdapterModelConfig(TransformersModelConfig): - """Configuration class for PEFT (Parameter-Efficient Fine-Tuning) adapter models. - - This configuration is used to load models that have been fine-tuned using PEFT adapters, - such as LoRA, AdaLoRA, or other parameter-efficient fine-tuning methods. The adapter - weights are merged with the base model during loading for efficient inference. - - Attributes: - base_model (str): - HuggingFace Hub model ID or path to the base model. This is the original - pre-trained model that the adapter was trained on. - - Note: - - Requires the `peft` library to be installed, `pip install lighteval[adapters]` - - Adapter models have the specificity that they look at the base model (= the parent) for the tokenizer and config - """ - - base_model: str - - -class AdapterModel(TransformersModel): - def _create_auto_model(self) -> transformers.PreTrainedModel: - """Returns a PeftModel from a base model and a version fined tuned using PEFT.""" - torch_dtype = _get_dtype(self.config.dtype) - model_parallel, max_memory, device_map = self.init_model_parallel(self.config.model_parallel) - self.config.model_parallel = model_parallel - - adapter_weights = self.config.pretrained - merged_path = f"{adapter_weights}-adapter-applied" - - if self.config.dtype == "4bit": - from transformers import BitsAndBytesConfig - - quantization_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.float16) - elif self.config.dtype == "8bit": - from transformers import BitsAndBytesConfig - - quantization_config = BitsAndBytesConfig(load_in_8bit=True) - else: - quantization_config = None - - if self.accelerator.is_local_main_process if self.accelerator is not None else nullcontext(): - logger.info(f"Loading model from {adapter_weights} and applying adapter to {self.config.base_model}") - base = AutoModelForCausalLM.from_pretrained( - self.config.base_model, torch_dtype=torch.float16, low_cpu_mem_usage=True - ) - # resize model for adapters with added tokens - token_diff = len(self._tokenizer) - base.config.vocab_size - if token_diff != 0: - if token_diff > 0: - logger.info( - f"You're using the adapter model's tokenizer, which has more tokens than the base model. Adding {token_diff} token(s)." - ) - else: - logger.info( - f"You're using the adapter model's tokenizer, which has fewer tokens than the base model. Removing {abs(token_diff)} token(s)." - ) - base.resize_token_embeddings(len(self._tokenizer)) - # Should pass revision - model = PeftModel.from_pretrained(base, adapter_weights) - model = model.merge_and_unload() - - logger.info("Saving model with adapter applied") - base.save_pretrained(merged_path) - - logger.info(f"Loading model from {merged_path}") - - model = AutoModelForCausalLM.from_pretrained( - merged_path, - max_memory=max_memory, - device_map=device_map, - torch_dtype=torch_dtype, - trust_remote_code=self.config.trust_remote_code, - quantization_config=quantization_config, - ) - - return model - - def cleanup(self): - try: - tmp_weights_dir = f"{self.model_name}-adapter-applied" - shutil.rmtree(tmp_weights_dir) - logger.info(f"Removed {tmp_weights_dir}") - except OSError: - pass diff --git a/src/lighteval/models/transformers/delta_model.py b/src/lighteval/models/transformers/delta_model.py deleted file mode 100644 index 3fb166c1c..000000000 --- a/src/lighteval/models/transformers/delta_model.py +++ /dev/null @@ -1,109 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import logging -import shutil -from contextlib import nullcontext - -import torch -from tqdm import tqdm -from transformers import AutoModelForCausalLM - -from lighteval.models.transformers.transformers_model import TransformersModel, TransformersModelConfig -from lighteval.models.utils import _get_dtype, _get_model_sha - - -logger = logging.getLogger(__name__) - - -class DeltaModelConfig(TransformersModelConfig): - """Configuration class for delta models (weight difference models). - - This configuration is used to load models that represent the difference between a - fine-tuned model and its base model. The delta weights are added to the base model - during loading to reconstruct the full fine-tuned model. - - Attributes: - base_model (str): - HuggingFace Hub model ID or path to the base model. This is the original - pre-trained model that the delta was computed from. - """ - - # Delta models look at the pretrained (= the delta weights) for the tokenizer and model config - base_model: str - - def get_model_sha(self): - return _get_model_sha(repo_id=self.model_name, revision="main") - - -class DeltaModel(TransformersModel): - def _create_auto_model( - self, - config: DeltaModelConfig, - ) -> AutoModelForCausalLM: - """Returns a model created by adding the weights of a delta model to a base model.""" - config.model_parallel, max_memory, device_map = self.init_model_parallel(config.model_parallel) - torch_dtype = _get_dtype(config.dtype, self._config) - - delta_model = config.pretrained - - merged_path = f"{delta_model}-delta-applied" - - if self.accelerator.is_main_process if self.accelerator is not None else nullcontext(): - logger.info(f"Loading base and delta models from {config.base_model} and {delta_model}") - base = AutoModelForCausalLM.from_pretrained( - config.base_model, torch_dtype=torch.float16, low_cpu_mem_usage=True - ) - delta = AutoModelForCausalLM.from_pretrained( - delta_model, - revision=config.revision + (f"/{config.subfolder}" if config.subfolder is not None else ""), - torch_dtype=torch.float16, - low_cpu_mem_usage=True, - ) - - for name, param in tqdm(base.state_dict().items(), desc="Applying delta"): - assert name in delta.state_dict() - param.data += delta.state_dict()[name] - - logger.info("Saving delta-applied model") - base.save_pretrained(merged_path) - - logger.info(f"Loading delta-applied model from {delta_model}-delta-applied") - - model = AutoModelForCausalLM.from_pretrained( - merged_path, - max_memory=max_memory, - device_map=device_map, - torch_dtype=torch_dtype, - trust_remote_code=config.trust_remote_code, - quantization_config=config.quantization_config, - ) - - return model - - def cleanup(self): - try: - tmp_weights_dir = f"{self.model_name}-delta-applied" - shutil.rmtree(tmp_weights_dir) - logger.info(f"Removed {tmp_weights_dir}") - except OSError: - pass diff --git a/src/lighteval/models/transformers/transformers_model.py b/src/lighteval/models/transformers/transformers_model.py deleted file mode 100644 index 3c19e6515..000000000 --- a/src/lighteval/models/transformers/transformers_model.py +++ /dev/null @@ -1,1284 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import logging -import os -from datetime import timedelta -from typing import Dict, Optional, Tuple, Union - -import torch -import torch.nn.functional as F -import transformers -from accelerate import Accelerator, InitProcessGroupKwargs -from accelerate.utils import calculate_maximum_sizes, convert_bytes, get_max_memory -from pydantic import Field, PositiveInt -from torch.nn.utils.rnn import pad_sequence -from torch.utils.data import DataLoader -from tqdm import tqdm -from transformers import ( - AutoConfig, - AutoModelForCausalLM, - AutoTokenizer, - BitsAndBytesConfig, - PretrainedConfig, -) -from transformers.generation.configuration_utils import GenerationConfig -from transformers.generation.utils import GenerateOutput - -from lighteval.data import GenerativeTaskDataset, LoglikelihoodDataset -from lighteval.models.abstract_model import LightevalModel, ModelConfig -from lighteval.models.model_output import ( - Batch, - ModelResponse, -) -from lighteval.models.utils import _get_dtype, _get_model_sha, _simplify_name, uses_chat_template -from lighteval.tasks.prompt_manager import PromptManager -from lighteval.tasks.requests import Doc, SamplingMethod -from lighteval.utils.cache_management import SampleCache, cached -from lighteval.utils.imports import ( - is_package_available, -) -from lighteval.utils.parallelism import find_executable_batch_size - - -logger = logging.getLogger(__name__) - -os.environ["TOKENIZERS_PARALLELISM"] = "false" - -STARTING_BATCH_SIZE = 512 - -# Thread local param -torch.set_grad_enabled(False) - - -class TransformersModelConfig(ModelConfig): - """Configuration class for HuggingFace Transformers models. - - This configuration is used to load and configure models from the HuggingFace Transformers library. - - Attributes: - model_name (str): - HuggingFace Hub model ID or path to a pre-trained model. This corresponds to the - `pretrained_model_name_or_path` argument in HuggingFace's `from_pretrained` method. - tokenizer (str | None): - Optional HuggingFace Hub tokenizer ID. If not specified, uses the same ID as model_name. - Useful when the tokenizer is different from the model (e.g., for multilingual models). - subfolder (str | None): - Subfolder within the model repository. Used when models are stored in subdirectories. - revision (str): - Git revision of the model to load. Defaults to "main". - batch_size (PositiveInt | None): - Batch size for model inference. If None, will be automatically determined. - max_length (PositiveInt | None): - Maximum sequence length for the model. If None, uses model's default. - model_loading_kwargs (dict): - Additional keyword arguments passed to `from_pretrained`. Defaults to empty dict. - add_special_tokens (bool): - Whether to add special tokens during tokenization. Defaults to True. - skip_special_tokens (bool): - Whether the tokenizer should output special tokens back during generation. Needed for reasoning models. Defaults to True - model_parallel (bool | None): - Whether to use model parallelism across multiple GPUs. If None, automatically - determined based on available GPUs and model size. - dtype (str | None): - Data type for model weights. Can be "float16", "bfloat16", "float32", "auto", "4bit", "8bit". - If "auto", uses the model's default dtype. - device (Union[int, str]): - Device to load the model on. Can be "cuda", "cpu", or GPU index. Defaults to "cuda". - trust_remote_code (bool): - Whether to trust remote code when loading models. Defaults to False. - compile (bool): - Whether to compile the model using torch.compile for optimization. Defaults to False. - multichoice_continuations_start_space (bool | None): - Whether to add a space before multiple choice continuations. If None, uses model default. - True forces adding space, False removes leading space if present. - pairwise_tokenization (bool): - Whether to tokenize context and continuation separately or together. Defaults to False. - continuous_batching (bool): - Whether to use continuous batching for generation. Defaults to False. - override_chat_template (bool): - If True, we force the model to use a chat template. If alse, we prevent the model from using - a chat template. If None, we use the default (true if present in the tokenizer, false otherwise) - generation_parameters (GenerationParameters, optional, defaults to empty GenerationParameters): - Configuration parameters that control text generation behavior, including - temperature, top_p, max_new_tokens, etc. - system_prompt (str | None, optional, defaults to None): Optional system prompt to be used with chat models. - This prompt sets the behavior and context for the model during evaluation. - cache_dir (str, optional, defaults to "~/.cache/huggingface/lighteval"): Directory to cache the model. - - Example: - ```python - config = TransformersModelConfig( - model_name="meta-llama/Llama-3.1-8B-Instruct", - batch_size=4, - dtype="float16", - generation_parameters=GenerationParameters( - temperature=0.7, - max_new_tokens=100 - ) - ) - ``` - - Note: - This configuration supports quantization (4-bit and 8-bit) through the dtype parameter. - When using quantization, ensure you have the required dependencies installed - (bitsandbytes for 4-bit/8-bit quantization). - """ - - model_name: str - tokenizer: str | None = None - subfolder: str | None = None - revision: str = "main" - batch_size: PositiveInt | None = None - max_length: PositiveInt | None = None - model_loading_kwargs: dict = Field(default_factory=dict) - add_special_tokens: bool = True - skip_special_tokens: bool = True - model_parallel: bool | None = None - dtype: str | None = None - device: Union[int, str] = "cuda" - trust_remote_code: bool = False - compile: bool = False - multichoice_continuations_start_space: bool | None = None - pairwise_tokenization: bool = False - continuous_batching: bool = False - override_chat_template: bool = None - - def model_post_init(self, __context): - if self.multichoice_continuations_start_space is True: - logger.warning( - "You set `multichoice_continuations_start_space` to true. This will force multichoice continuations to use a starting space" - ) - if self.multichoice_continuations_start_space is False: - logger.warning( - "You set `multichoice_continuations_start_space` to false. This will remove a leading space from multichoice continuations, if present." - ) - - def get_transformers_config(self) -> PretrainedConfig: - revision = self.revision - - if self.subfolder: - revision = f"{self.revision}/{self.subfolder}" - - auto_config = AutoConfig.from_pretrained( - self.model_name, - revision=revision, - trust_remote_code=self.trust_remote_code, - ) - - return auto_config - - def get_model_sha(self): - return _get_model_sha(repo_id=self.model_name, revision=self.revision) - - -class TransformersModel(LightevalModel): - def __init__( - self, - config: TransformersModelConfig, - ): - """Initializes a HuggingFace `AutoModel` and `AutoTokenizer` for evaluation.""" - self.config = config - self.accelerator = Accelerator(kwargs_handlers=[InitProcessGroupKwargs(timeout=timedelta(seconds=3000))]) - self._device = self.accelerator.device - self.multichoice_continuations_start_space = config.multichoice_continuations_start_space - self._add_special_tokens = config.add_special_tokens or False - self.skip_special_tokens = config.skip_special_tokens or True - self.pairwise_tokenization = config.pairwise_tokenization - self.batch_size = config.batch_size - self.continuous_batching = config.continuous_batching - self.transformers_config = config.get_transformers_config() - self.generation_config_dict = config.generation_parameters.to_transformers_dict() - - self.model_sha = config.get_model_sha() - self._max_length = self._init_max_length() - self._tokenizer = self._create_auto_tokenizer() - self.use_chat_template = uses_chat_template( - tokenizer=self._tokenizer, override_chat_template=config.override_chat_template - ) - self.model = self._create_auto_model() - - # We are in DP (and launch the script with `accelerate launch`) - if config.model_parallel is False and self.config.dtype not in ["4bit", "8bit"]: - logger.info(f"Using Data Parallelism, putting model on device {self._device}") - self.model = self.model.to(self._device) - - self.model_name = _simplify_name(config.model_name) - - if is_package_available("accelerate"): - model_size, _ = calculate_maximum_sizes(self.model) - model_size = convert_bytes(model_size) - else: - model_size = -1 - - self.prompt_manager = PromptManager( - use_chat_template=self.use_chat_template, tokenizer=self.tokenizer, system_prompt=config.system_prompt - ) - - # Initialize cache for tokenization and predictions - self._cache = SampleCache(config) - - def cleanup(self): - """Clean up operations if needed, such as closing an endpoint.""" - del self.model - del self._tokenizer - torch.cuda.empty_cache() - - @classmethod - def from_model( - cls, - model: AutoModelForCausalLM, - config: TransformersModelConfig, - accelerator: Accelerator | None = None, - ) -> "TransformersModel": - if config is None: - raise ValueError("Config must be provided to initialize the TransformersModel via `from_model` method.") - - # Instanciate the object without using __init__ - self = cls.__new__(cls) - - self.transformers_config = model.config - - self.config = config - self.multichoice_continuations_start_space = config.multichoice_continuations_start_space - self._add_special_tokens = config.add_special_tokens - self.skip_special_tokens = config.skip_special_tokens - self.pairwise_tokenization = config.pairwise_tokenization - self.batch_size = config.batch_size - self.continuous_batching = config.continuous_batching - self.generation_config_dict = config.generation_parameters.to_transformers_dict() - - self.model_name = config.model_name - self.model_sha = config.get_model_sha() - self._max_length = self._init_max_length() - self._tokenizer = self._create_auto_tokenizer() - self.use_chat_template = uses_chat_template( - tokenizer=self._tokenizer, override_chat_template=config.override_chat_template - ) - - # If model_parallel is not set we compare the number of processes with the number of GPUs - self.model = model - self.model.eval() - torch.set_grad_enabled(False) - - self.accelerator = accelerator - if accelerator is not None: - self._device = accelerator.device - self.model = self.accelerator.prepare(self.model.to(accelerator.device)) - else: - self._device = self.config.device - - if is_package_available("accelerate"): - model_size, _ = calculate_maximum_sizes(self.model) - model_size = convert_bytes(model_size) - else: - model_size = -1 - self.prompt_manager = PromptManager( - use_chat_template=self.use_chat_template, - tokenizer=self.tokenizer, - system_prompt=config.system_prompt if config else None, - ) - - # Initialize cache for tokenization and predictions - self._cache = SampleCache(config) if config else None - - return self - - @property - def tokenizer(self): - return self._tokenizer - - @property - def add_special_tokens(self): - return self._add_special_tokens - - @property - def max_length(self) -> int: - return self._max_length - - @property - def device(self) -> Union[str, torch.device]: - return self._device - - @property - def disable_tqdm(self) -> bool: - disable_tqdm = False - if self.accelerator: - disable_tqdm = bool(not self.accelerator.is_main_process) - return disable_tqdm - - def init_model_parallel(self, model_parallel: bool | None = None) -> Tuple[bool, Optional[dict], Optional[str]]: - """Compute all the parameters related to model_parallel""" - if not is_package_available("accelerate"): - return False, None, None - - self.num_local_processes = int(os.environ.get("LOCAL_WORLD_SIZE", 1)) - self.num_machines = torch.cuda.device_count() // self.num_local_processes - - if self.num_machines == 1: - logger.info("We are not in a distributed setting. Setting model_parallel to False.") - model_parallel = False - - if model_parallel is None: - max_memory_all_gpus = get_max_memory() # A dict of the max memory for all the gpus - if "cpu" in max_memory_all_gpus: - del max_memory_all_gpus["cpu"] - model_parallel = bool(self.num_local_processes < len(max_memory_all_gpus)) - logger.info( - f"Setting model parallel to {model_parallel} since " - f"the number of local processes is {self.num_local_processes} " - f"and the number of GPUs is {len(max_memory_all_gpus)}" - ) - if model_parallel is True: - max_memory_all_gpus = get_max_memory() # A dict of the max memory for all the gpus - if "cpu" in max_memory_all_gpus: - del max_memory_all_gpus["cpu"] - max_mem_this_process = { - k: v - for k, v in max_memory_all_gpus.items() - if k % self.num_local_processes == (self.accelerator.process_index % self.num_local_processes) - } - device_map = "auto" - logger.info( - f"Model parallel was set to True, setting max memory per GPU to {max_mem_this_process} and device map to {device_map}" - ) - else: - max_mem_this_process = None - device_map = None - logger.info( - f"Model parallel was set to False, max memory set to {max_mem_this_process} and device map to {device_map}" - ) - return model_parallel, max_mem_this_process, device_map - - def _create_auto_model(self) -> transformers.PreTrainedModel: - """Creates an instance of the pretrained HF model. - - Returns: - transformers.PreTrainedModel: The created auto model instance. - """ - model_parallel, max_memory, device_map = self.init_model_parallel(self.config.model_parallel) - self.config.model_parallel = model_parallel - - if self.config.dtype == "4bit": - quantization_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.float16) - elif self.config.dtype == "8bit": - quantization_config = BitsAndBytesConfig(load_in_8bit=True) - else: - quantization_config = None - - torch_dtype = _get_dtype(self.config.dtype) - subfolder = self.config.subfolder - revision = self.config.revision + (f"/{subfolder}" if subfolder is not None else "") - - pretrained_config = self.transformers_config - - kwargs = self.config.model_loading_kwargs.copy() - if "quantization_config" not in pretrained_config.to_dict(): - kwargs["quantization_config"] = quantization_config - - model = AutoModelForCausalLM.from_pretrained( - self.config.model_name, - revision=revision, - max_memory=max_memory, - device_map=device_map, - # tp_plan="auto", - torch_dtype=torch_dtype, - trust_remote_code=self.config.trust_remote_code, - **kwargs, - ) - # model.to(self.device) - model.eval() - - if self.continuous_batching: - generation_config = GenerationConfig( - **self.generation_config_dict, - ) - model.generation_config = generation_config - - if self.config.compile: - try: - logger.info("Compiling the model") - model.compile() - except AttributeError as e: - logger.warning("Could not compile the model because: ", e) - - return model - - def _create_auto_tokenizer( - self, - ) -> transformers.PreTrainedTokenizer: - """Create a Hugging Face AutoTokenizer for language model. - - Returns: - transformers.PreTrainedTokenizer: The created tokenizer. - """ - tokenizer_name = self.config.tokenizer or self.config.model_name - subfolder = self.config.subfolder - revision = self.config.revision + (f"/{subfolder}" if subfolder is not None else "") - - tokenizer = AutoTokenizer.from_pretrained( - tokenizer_name, - revision=revision, - trust_remote_code=self.config.trust_remote_code, - padding_side="left", - truncation_side="left", - ) - tokenizer.pad_token = tokenizer.eos_token - tokenizer.model_max_length = self.max_length - logger.info("Tokenizer truncation and padding size set to the left side.") - - return tokenizer - - def _init_max_length(self) -> int: - """Return the maximum sequence length of the model. - NOTE: Different model configurations have different max sequence length - attribute names. - - n_positions: (CTRLConfig) - - max_position_embeddings: (BartConfig, RoFormerConfig) - - n_ctx: (GPT2Config) - NOTE: For relative position encoded models you should specify the max - sequence length of the model in the constructor via `max_length`. - - Returns: - int: Max length to use depending on the available args and config - """ - if self.config.max_length is not None: - return self.config.max_length - - # Try to get the sequence length from the model config. - text_model_config = self.transformers_config.get_text_config() - - seqlen_config_attrs = ("n_positions", "max_position_embeddings", "n_ctx") - for attr in seqlen_config_attrs: - if hasattr(text_model_config, attr): - return getattr(text_model_config, attr) - - logger.warning( - "No max_length attribute found in the model config. Using the default max sequence length setting `2048`. " - "It is recommended to set max_length trough the model args: max_length=..." - ) - - return 2048 - - def _check_continuations_start_space(self, continuation: str) -> str: - """Some models tokenizer want a space at the beginning and other not. We update this if needed here. - multichoice_continuations_start_space can be: - - True (add a space if these isn't one) - - False (remove a space if there is one) - - None (Don't touch - default) - Todo: find a way to add this back WITHOUT breaking compatibility with the harness - """ - if self.multichoice_continuations_start_space is not None: - if self.multichoice_continuations_start_space and continuation[0] != " ": - continuation = " " + continuation - if not self.multichoice_continuations_start_space and continuation[0] == " ": - continuation = continuation.lstrip() - return continuation - - def _get_batch_size(self, max_input_length: int, override_bs: int | None, starting_batch_size: int = 512) -> int: - if override_bs is not None: - return override_bs - logger.info(f"Detecting largest batch size with max_input_length={max_input_length}") - - @find_executable_batch_size( - starting_batch_size=starting_batch_size - ) # if OOM, then halves batch_size and tries again - def forward_batch(batch_size): - fake_batch, fake_output = None, None - with torch.no_grad(): - try: - fake_batch = torch.ones((batch_size, max_input_length), device=self.device).int() - fake_output = F.log_softmax(self.model(fake_batch).logits, dim=-1).cpu() - except Exception as e: - for fake_item in [fake_batch, fake_output]: - if fake_item is not None: - fake_item.detach() - del fake_item - - raise e - return batch_size - - batch_size = forward_batch() - logger.info(f"Determined largest batch size: {batch_size}") - return batch_size - - def _continuous_greedy_until( - self, - docs: list[Doc], - ) -> list[ModelResponse]: - """Generates responses using a greedy decoding strategy until certain ending conditions are met. - - Args: - docs (list[Doc]): List of documents containing the context for generation. - - Returns: - list[ModelResponse]: list of generated responses. - """ - dataset = GenerativeTaskDataset(requests=docs, num_dataset_splits=self.DATASET_SPLITS) - results = [] - - for split in tqdm( - dataset.splits_iterator(), - total=dataset.num_dataset_splits, - desc="Splits", - position=0, - disable=False, # self.disable_tqdm, - ): - # For chat models, generation stops with EOS token, so we don't need to specify stop tokens - if self.use_chat_template: - stop_tokens = [] - else: - # NOTE: we are assuming all items in a batch behave similarly (same - # stop_tokens and max_tokens genrated) which is not necessarily - # the case! Because of that we only use batch size of 1 - stop_tokens = split[0].stop_sequence - - max_new_tokens = self.config.generation_parameters.max_new_tokens or split[0].generation_size - returns_logits = split[0].use_logits - num_samples = split[0].num_samples - contexts = [self.prompt_manager.prepare_prompt(doc) for doc in split] - tokenized = self.tokenizer(contexts, add_special_tokens=self.add_special_tokens) - - # The main question for this step is the following: - # Would we rather truncate the prompt to allow generation to go to max_new_tokens, at the risk - # of losing some meaning, or have some generations that are exceedingly short? - # The choice we go for here is to avoid truncating the prompt if we can, since it - # should have been managed by the prompt creator/few shot manager if requested by the user. - inputs = tokenized["input_ids"] - context_size = len(inputs[0]) - - # left truncate the inputs to the maximum length - if max_new_tokens is not None: - if context_size + max_new_tokens > self.max_length: - logger.warning( - f"{context_size + max_new_tokens=} which is greater than {self.max_length=}. Truncating context to {self.max_length - max_new_tokens} tokens." - ) - context_size = self.max_length - max_new_tokens - if context_size < 0: - logger.critical( - f"{context_size=} is less than 0, either reduce the max_new_tokens or increase model max length." - ) - raise ValueError("Context size is less than 0.") - inputs = [input[-context_size:] for input in inputs] - else: - if context_size > self.max_length: - logger.warning( - f"{context_size=} which is greater than {self.max_length=}. Truncating context to {self.max_length} tokens." - ) - context_size = self.max_length - inputs = [input[-context_size:] for input in inputs] - - _outputs = self._generate( - inputs=inputs, - max_new_tokens=max_new_tokens, - stop_tokens=stop_tokens, - returns_logits=returns_logits, - num_samples=num_samples, - continuous_batching=True, - ) - - for req_id, _output in _outputs.items(): - output_token_ids = [] - logprobs_raw = [] - result = [] - - # for output in _output.outputs: - output_token_ids.append(_output.generated_tokens) - # logprobs_raw.append(output.logprobs) - result.append( - self.tokenizer.decode(_output.generated_tokens, skip_special_tokens=self.skip_special_tokens) - ) - - if logprobs_raw and output_token_ids and False: - logprobs = [logprobs_raw[0][token_id].logprob for token_id in output_token_ids[0]] - else: - logprobs = [] - - input_token_ids = _output.prompt_ids - cur_response = ModelResponse( - text=result, - logprobs=logprobs, - output_tokens=output_token_ids, - input_tokens=input_token_ids, - ) - results.append(cur_response) - - return dataset.get_original_order(results) - - def _padded_greedy_until( - self, - docs: list[Doc], - ) -> list[ModelResponse]: - """Generates responses using a greedy decoding strategy until certain ending conditions are met. - - Args: - docs (list[Doc]): List of documents containing the context for generation. - - Returns: - list[ModelResponse]: list of generated responses. - """ - results = [] - dataset = GenerativeTaskDataset(requests=docs, num_dataset_splits=self.DATASET_SPLITS) - starting_batch_size = STARTING_BATCH_SIZE - - for split in tqdm( - dataset.splits_iterator(), - total=dataset.num_dataset_splits, - desc="Splits", - position=0, - disable=self.disable_tqdm, - ): - if self.generation_config_dict.get("max_new_tokens", None) is not None: - # The user forces a specific generation size - max_context_continuation_size_allowed = self.generation_config_dict["max_new_tokens"] - elif split[0].generation_size is None: - # No constraints on the generation size: max length allowed is the max model context - max_context_continuation_size_allowed = self.max_length - else: - # The task forces a specific generation size - context = self.prompt_manager.prepare_prompt(split[0]) - tokenized_context = self.tokenizer(context) - - # Longest context in the current split is the first item (since we sort reversed) - longest_context_continuation_size_in_split = ( - len(tokenized_context["input_ids"]) + split[0].generation_size - ) - max_context_continuation_size_allowed = min( - longest_context_continuation_size_in_split, self.max_length - ) - batch_size = self._get_batch_size( - override_bs=self.config.batch_size, - max_input_length=max_context_continuation_size_allowed, - starting_batch_size=starting_batch_size, - ) - # For next iteration, since the batch will be smaller, we'll test a bigger batch size - starting_batch_size = batch_size * 2 - - dataloader = DataLoader(split, batch_size=batch_size, collate_fn=lambda batch: batch) - if self.accelerator: - dataloader = self.accelerator.prepare(dataloader) - - for batch in tqdm( - dataloader, desc="Greedy generation", position=1, leave=False, disable=self.disable_tqdm - ): - contexts = [self.prompt_manager.prepare_prompt(doc) for doc in batch] - - # For chat models, generation stops with EOS token, so we don't need to specify stop tokens - if self.use_chat_template: - stop_tokens = [self.tokenizer.eos_token] - else: - # NOTE: we are assuming all items in a batch behave similarly (same - # stop_tokens and max_tokens genrated) which is not necessarily - # the case! Because of that we only use batch size of 1 - stop_tokens = [self.tokenizer.eos_token] + batch[0].stop_sequences - - max_new_tokens = batch[0].generation_size - num_samples = batch[0].num_samples - - # See doc https://huggingface.co/docs/transformers/v4.38.2/en/pad_truncation#padding-and-truncation - # Will do left truncation and padding, as defined when creating the tokenizer - tokenized = self.tokenizer( - contexts, - truncation="longest_first", # we truncate to the model max length if needed - padding="longest", # we pad to the longest sequence - return_tensors="pt", - max_length=max_context_continuation_size_allowed, # we always allow minimum one token of generation - add_special_tokens=self.add_special_tokens, - ).to(self.device) - - # The main question for this step is the following: - # Would we rather truncate the prompt to allow generation to go to max_new_tokens, at the risk - # of losing some meaning, or have some generations that are exceedingly short? - # The choice we go for here is to avoid truncating the prompt if we can, since it - # should have been managed by the prompt creator/few shot manager if requested by the user. - context_size = tokenized["input_ids"].shape[1] - if context_size > self.max_length: - logger.warning( - f"The context size of your batch ({context_size}) is bigger than the maximum context size allowed by the model ({self.max_length}) for a task in" - + str({i.task_name for i in batch}) - + ". This is likely to lead to some errors." # noqa C401 - ) - # There will be truncation of at least one sample, maximum generation size will be one - max_new_tokens = 1 - else: # We can't allow generation of more than max_length - if max_new_tokens is None: # If generation size is not set, we go all the way - max_new_tokens = self.max_length - context_size - else: - max_new_tokens = min(self.max_length - context_size, max_new_tokens) - if max_new_tokens < 1: - max_new_tokens = 1 - - prepared_batch = Batch( - input_ids=tokenized["input_ids"], - input_lengths=[len(item == 1) for item in tokenized["attention_mask"]], - input_mask=tokenized["attention_mask"], - truncated=[max(len(c) - tokenized["input_ids"].shape[1], 0) for c in contexts], - padded=[sum(mask == 0) for mask in tokenized["attention_mask"]], - ) - - cur_reponses = self._generate( - batch=prepared_batch, - max_new_tokens=max_new_tokens, - stop_tokens=stop_tokens, - returns_logits=False, - num_samples=num_samples, - continuous_batching=False, - ) - results.extend(cur_reponses) - - return dataset.get_original_order(results) - - @cached(SamplingMethod.GENERATIVE) - def greedy_until( - self, - docs: list[Doc], - ) -> list[ModelResponse]: - if self.continuous_batching: - return self._continuous_greedy_until(docs) - else: - return self._padded_greedy_until(docs) - - def _generate_continuous( - self, - inputs: list[list[int]], - max_new_tokens: Optional[int] = None, - stop_tokens: Optional[list[str]] = None, - returns_logits: Optional[bool] = False, - num_samples: int = 1, - generate: bool = True, - ) -> Dict[str, ModelResponse]: - # Compute model generation - self.model.generation_config.use_cuda_graph = False # Disable CUDA graph for batch generation - self.model.generation_config.max_batch_tokens = 256 # Disable CUDA graph for batch generation - # self.model.generation_config.do_sample = False # Disable CUDA graph for batch generation - batch_outputs = self.model.generate_batch( - inputs=inputs, - generation_config=self.model.generation_config, - # You can pass request-specific overrides here, e.g., max_new_tokens=100 - ) - - return batch_outputs - - def _generate_padded( - self, - batch: Batch, - max_new_tokens: int, - stop_tokens: list[str], - returns_logits: Optional[bool] = False, - num_samples: int = 1, - ) -> list[ModelResponse]: - """Contains the actual logic of the generation. - First computes the stop sequences, then generates the predictions, then converts the outputs to ModelResponse. - """ - stopping_criteria = stop_sequences_criteria(self.tokenizer, stop_sequences=stop_tokens, batch=batch) - batch_size, _ = batch.input_ids.shape - - if num_samples > 1 and self.generation_config_dict["temperature"] == 0: - raise ValueError( - "You cannot generate multiple samples with temperature=0. Please set temperature > 0. Or use a non sampling metric." - ) - - generation_config = self.generation_config_dict.copy() - generation_config.update( - max_new_tokens=max_new_tokens, - pad_token_id=self.tokenizer.pad_token_id if self.tokenizer.pad_token_id else self.tokenizer.eos_token_id, - eos_token_id=self.tokenizer.eos_token_id, - num_return_sequences=num_samples, - output_logits=returns_logits, - renormalize_logits=True, - ) - if num_samples == 1 and generation_config["temperature"] == 0: - generation_config["do_sample"] = False - if num_samples > 1 and generation_config["temperature"] == 0: - logger.warning("num_samples > 1 but temperature is set to 0, this will not sample different outputs.") - - # Compute model generation - outputs: GenerateOutput = self.model.generate( - input_ids=batch.input_ids, - attention_mask=batch.input_mask, - stopping_criteria=stopping_criteria, - **generation_config, - ) - generations = outputs.sequences[:, batch.input_ids.size(1) :] - generations = torch.reshape(generations, (batch_size, num_samples, -1)) - generations, len_gens = self.pad_and_gather(generations, num_samples=num_samples) - batch.input_ids, len_ids = self.pad_and_gather(batch.input_ids) - - logits, len_logits = None, None - if returns_logits: - logits, len_logits = self.pad_and_gather(outputs.logits) - logits = logits.cpu().numpy() - - # We gather remaining info - batch.truncated = torch.tensor(batch.truncated, device=self.device) - if self.accelerator: - batch.truncated = self.accelerator.gather_for_metrics(batch.truncated) - batch.padded = torch.tensor(batch.padded, device=self.device) - if self.accelerator: - batch.padded = self.accelerator.gather_for_metrics(batch.padded) - - # We convert to ModelResponse outputs - all_responses = [] - for ix, (batched_generations, batched_input, trunc, padded) in enumerate( - zip(generations, batch.input_ids, batch.truncated, batch.padded) - ): - result_generations = [] - decoded_generations = [] - # Ensure the generated responses do not contain the stop sequences. - for generation in batched_generations: - generation = generation[: len_gens[ix]] - result_generations.append(generation) - decoded_generation = self.tok_decode([generation])[0] - - for term in stop_tokens: - decoded_generation = decoded_generation.split(term)[0] - - decoded_generations.append(decoded_generation) - - cur_response = ModelResponse( - text=decoded_generations, - output_tokens=result_generations, - logits=logits[ix][: len_logits[ix]] if returns_logits else None, - input_tokens=batched_input[: len_ids[ix]].tolist(), - truncated_tokens_count=trunc.cpu().item(), - padded_tokens_count=padded.cpu().item(), - ) - all_responses.append(cur_response) - - return all_responses - - def _generate( - self, - continuous_batching: bool, - **kwargs, - ) -> list[ModelResponse]: - if continuous_batching: - return self._generate_continuous(**kwargs) - else: - return self._generate_padded(**kwargs) - - @cached(SamplingMethod.LOGPROBS) - def loglikelihood( - self, - docs: list[Doc], - ) -> list[ModelResponse]: - """Tokenize the context and continuation and compute the log likelihood of those - tokenized sequences. - - Args: - docs (list[Doc]): List of documents containing the context and choices. - - Returns: - list[ModelResponse]: List of model responses with logprobs. - """ - return self._loglikelihood_tokens(docs) - - @cached(SamplingMethod.PERPLEXITY) - def loglikelihood_rolling( - self, - docs: list[Doc], - ) -> list[ModelResponse]: - """This function is used to compute the log likelihood of the context for perplexity metrics.""" - return self._loglikelihood_tokens( - docs, - rolling=True, - ) - - def _loglikelihood_tokens( # noqa: C901 - self, - docs: list[Doc], - rolling=False, - ) -> list[ModelResponse]: - dataset = LoglikelihoodDataset(requests=docs, num_dataset_splits=1) - starting_batch_size = STARTING_BATCH_SIZE - all_responses = [] - - for split in tqdm(dataset.splits_iterator(), disable=self.disable_tqdm): - first_doc_context = self.prompt_manager.prepare_prompt(split[0]) - tokenized_contexts, tokenized_continuations = self.tok_encode_pair( - first_doc_context, split[0].choices, pairwise=self.pairwise_tokenization - ) - - if rolling: # we take all the sequence in rolling mode - max_input_length = len(tokenized_contexts[0] + tokenized_continuations[0]) - else: # in normal mode, we left cut the context if needed - max_input_length = max( - min(self.max_length, len(tokenized_contexts[0] + tokenized_continuations[0]) - 1), 1 - ) - - batch_size = self._get_batch_size( - override_bs=self.config.batch_size, - max_input_length=max_input_length, - starting_batch_size=starting_batch_size, - ) - starting_batch_size = batch_size * 2 - max_num_choices = max(len(d.choices) for d in split) - # We divide the batch size by the number of choices as batch is samples * num choices - # then round up to closest 8 multiple - batch_size = max(1, round(batch_size // max_num_choices / 8) * 8) - logger.warning( - f"batch size is set to {batch_size} (it should be understood as '{batch_size} times the maximum number of choices per sample, {max_num_choices}')" - ) - - dataloader = DataLoader(split, batch_size=batch_size, collate_fn=lambda batch: batch) - if self.accelerator: - dataloader = self.accelerator.prepare(dataloader) - - for batch in tqdm(dataloader, disable=self.disable_tqdm): - batch_contexts: list[str] = [self.prompt_manager.prepare_prompt(doc) for doc in batch] - batch_tokenized_contexts = [] - batch_tokenized_continuations = [] - - for context, doc in zip(batch_contexts, batch): - doc_contexts, doc_continuations = self.tok_encode_pair(context, doc.choices, pairwise=True) - batch_tokenized_contexts.append(doc_contexts) - batch_tokenized_continuations.append(doc_continuations) - - prepared_batch = self.prepare_batch_logprob( - tokenized_contexts=batch_tokenized_contexts, - tokenized_continuations=batch_tokenized_continuations, - max_context=None, # computed as model max length in the function - ) - - model_output = self.model(prepared_batch.input_ids).logits - logits = F.log_softmax(model_output, dim=-1) # [batch, sequence_length, vocab] - - flat_index = 0 - batch_logits_sums = [] - batch_max_equals = [] - batch_tokenized_contexts_processed = [] - batch_tokenized_continuations_processed = [] - batch_choices_counts = [] - - # Flatten the logits to match the number of choices per sample - for doc_idx, doc in enumerate(batch): - # Get the size of the corresponding nested list - num_choices = len(batch_tokenized_continuations[doc_idx]) - doc_continuations = batch_tokenized_continuations[doc_idx] - # Extract the corresponding elements from flat_values - doc_logits = logits[flat_index : flat_index + num_choices] - doc_input_lengths = prepared_batch.input_lengths[flat_index : flat_index + num_choices] - # Move the index forward - flat_index += num_choices - - doc_logits_sums = [] - doc_max_equals = [] - doc_continuation_tokens = [] - - for choice_logits, input_length, choice_continuation in zip( - doc_logits, doc_input_lengths, doc_continuations - ): - choice_continuation_tensor = torch.tensor( - choice_continuation, dtype=torch.long, device=self.device - ) - continuation_length = len(choice_continuation_tensor) - if rolling: - choice_logits = choice_logits.unsqueeze(0).to(self.device) # [1, seq, vocab] - choice_continuation_tensor = ( - choice_continuation_tensor[:input_length].unsqueeze(0).to(self.device) - ) # [1, seq] - else: - choice_logits = ( - choice_logits[input_length - continuation_length - 1 : input_length - 1] - .unsqueeze(0) - .to(self.device) - ) - choice_continuation_tensor = choice_continuation_tensor.unsqueeze(0).to( - self.device - ) # [1, seq] - - # Check if per-token argmax is exactly equal to continuation - greedy_tokens = choice_logits.argmax(dim=-1).to(self.device) - # Sometimes the continuation is longer than allowed by the model, we only look at the first tokens - is_exact_match = (greedy_tokens == choice_continuation_tensor).all().squeeze(0).to(self.device) - - # Obtain log-probs at the corresponding continuation token indices - choice_logits = torch.gather( - choice_logits, 2, choice_continuation_tensor.unsqueeze(-1) - ).squeeze(-1) # [1, seq] - - # Answer: (log prob, is-exact-match) - doc_logits_sums.append(choice_logits.sum()) - doc_max_equals.append(is_exact_match) - doc_continuation_tokens.append(choice_continuation_tensor.squeeze(0)) # [seq] - - batch_logits_sums.append(torch.stack(doc_logits_sums)) - batch_max_equals.append(torch.stack(doc_max_equals)) - batch_tokenized_continuations_processed.append( - pad_sequence(doc_continuation_tokens, batch_first=True, padding_value=-1) - ) - batch_tokenized_contexts_processed.append( - torch.tensor(batch_tokenized_contexts[doc_idx][0], dtype=torch.long, device=self.device) - ) - batch_choices_counts.append( - torch.tensor(len(batch_tokenized_continuations[doc_idx]), dtype=torch.long, device=self.device) - ) - - # Gather the results from all processes - # need to gather logits_sum_batch, max_equals_batch and batch_cont_tokens_batch, contexts - if self.accelerator: - # Convert lists to tensors for proper gathering - # Pad and stack the tensors to make them gatherable - shape_choices = [ - choices.shape for choices in batch_tokenized_continuations_processed - ] # num_choices * max len choices - num_choices_tensor = torch.tensor([shape[0] for shape in shape_choices], device=self.device) - len_choices_tensor = torch.tensor([shape[1] for shape in shape_choices], device=self.device) - gathered_num_choices = self.accelerator.gather_for_metrics(num_choices_tensor) - gathered_len_choices = self.accelerator.gather_for_metrics(len_choices_tensor) - max_num_choices = gathered_num_choices.max().item() - max_len_choices = gathered_len_choices.max().item() - len_context_tensor = torch.tensor( - [len(ctx) for ctx in batch_tokenized_contexts_processed], device=self.device - ) - gathered_len_context = self.accelerator.gather_for_metrics(len_context_tensor) - max_len_context = gathered_len_context.max().item() - - # 1d - Pad logits_sum and max_equals to same number of choices - padded_logits_sums = [] - for logits_sum_doc in batch_logits_sums: - pad_amount = max_num_choices - len(logits_sum_doc) - padded = F.pad(logits_sum_doc, (0, pad_amount), value=-1) - padded_logits_sums.append(padded) - - padded_max_equals = [] - for max_equals_doc in batch_max_equals: - pad_amount = max_num_choices - len(max_equals_doc) - padded = F.pad(max_equals_doc, (0, pad_amount), value=False) - padded_max_equals.append(padded) - - # 2d - Pad continuations to max number of choice and max length - padded_continuations = [] - for cont_batch in batch_tokenized_continuations_processed: - pad_amount_num = max_num_choices - cont_batch.shape[0] - pad_amount_len = max_len_choices - cont_batch.shape[1] - padded = F.pad(cont_batch, (0, pad_amount_len, 0, pad_amount_num), value=-1) - padded_continuations.append(padded) - - # 1d - Pad context to maximum context size - padded_contexts = [] - for ctx_batch in batch_tokenized_contexts_processed: - pad_amount = max_len_context - len(ctx_batch) - padded = F.pad(ctx_batch, (0, pad_amount), value=-1) - padded_contexts.append(padded) - - # Stack all tensors for gathering - stacked_logits_sums = torch.stack(padded_logits_sums) - stacked_max_equals = torch.stack(padded_max_equals) - stacked_continuations = torch.stack(padded_continuations) - stacked_contexts = torch.stack(padded_contexts) - - # Gather the stacked tensors - gathered_logits_sums = self.accelerator.gather_for_metrics(stacked_logits_sums) - gathered_max_equals = self.accelerator.gather_for_metrics(stacked_max_equals) - gathered_continuations = self.accelerator.gather_for_metrics(stacked_continuations) - gathered_contexts = self.accelerator.gather_for_metrics(stacked_contexts) - - # Convert back to lists for processing - batch_logits_sums = [] - batch_max_equals = [] - batch_tokenized_continuations_processed = [] - batch_tokenized_contexts_processed = [] - - # Only process if we have gathered results - for i, num_choices in enumerate(gathered_num_choices): - # Extract non-padded values - # 1d on num choices - batch_logits_sums.append(gathered_logits_sums[i][:num_choices]) - batch_max_equals.append(gathered_max_equals[i][:num_choices]) - # 2d on num choices and max len - len_choice = gathered_len_choices[i] - batch_tokenized_continuations_processed.append( - gathered_continuations[i][:num_choices][:len_choice] - ) - # 1d on max len context - len_context = gathered_len_context[i] - batch_tokenized_contexts_processed.append(gathered_contexts[i][:len_context]) - - # Process the gathered results - for i in range(len(batch_logits_sums)): - max_equals_doc = batch_max_equals[i] - logits_sum_doc = batch_logits_sums[i] - tokenized_contexts_batch = batch_tokenized_contexts_processed[i] - tokenized_continuations_batch = batch_tokenized_continuations_processed[i] - answer = ModelResponse( - argmax_logits_eq_gold=[max_equal.cpu().item() for max_equal in max_equals_doc], - logprobs=[sum.cpu().item() for sum in logits_sum_doc], - input_tokens=tokenized_contexts_batch, - output_tokens=tokenized_continuations_batch, - ) - all_responses.append(answer) - - return dataset.get_original_order(all_responses) - - def prepare_batch_logprob( - self, - tokenized_contexts: list[list[list[int]]], - tokenized_continuations: list[list[list[int]]], - max_context: Optional[int] = None, - ): - """Tokenize a batch of inputs and return also the length, truncations and padding. - This step is done manually since we tokenize log probability inputs together with their continuation, - to manage possible extra spaces added at the start by tokenizers, see tok_encode_pair. - """ - inputs = [] - # we used to remove the last token of continuation, but it's not needed - for tokenized_context, tokenized_continuation in zip(tokenized_contexts, tokenized_continuations): - inputs.extend( - [context + continuation for context, continuation in zip(tokenized_context, tokenized_continuation)] - ) - - input_tokens = [] - attention_masks = [] - input_lengths = [] - truncated = [] - padded = [] - - if max_context is None: - max_context = self.max_length - - # First, find the longest sequence length in the batch - max_sequence_length = max(len(seq) for seq in inputs) - # If max_sequence_length is longer than max_context, we need to truncate - effective_max_length = min(max_sequence_length, max_context) - - # Each sample is concatenated and cut to length or padded to max_length - for orig_tokens in inputs: - # Calculate truncation - truncated.append(max(len(orig_tokens) - effective_max_length, 0)) - - # Truncate from the left if needed to fit in the model's context - tokens = torch.tensor((orig_tokens)[-effective_max_length:], dtype=torch.long).to(self.device) - sequence_len = tokens.shape[0] - - # Calculate padding needed to reach effective_max_length - padding_needed = effective_max_length - sequence_len - padded.append(padding_needed) - - # Right padding to reach effective_max_length - tokens = F.pad(tokens, (0, padding_needed), value=self.tokenizer.pad_token_id) - - # Create attention mask (1 for real tokens, 0 for padding) - mask = torch.ones_like(tokens, dtype=torch.bool) - mask[sequence_len:] = False - attention_masks.append(mask) - - input_tokens.append(tokens.unsqueeze(0)) # [1, effective_max_length] - input_lengths.append(sequence_len) - - batched_inputs = torch.cat(input_tokens, dim=0) # [batch, effective_max_length] - attention_masks = torch.cat(attention_masks, dim=0) # [batch, effective_max_length] - - return Batch( - input_ids=batched_inputs, - input_mask=attention_masks, - input_lengths=input_lengths, - truncated=truncated, - padded=padded, - ) - - def pad_and_gather( - self, output_tensor: torch.Tensor, drop_last_samples: bool = True, num_samples: int = None - ) -> torch.Tensor: - """Pads the `output_tensor` to the maximum length and gathers the lengths across processes. - - Args: - output_tensor (torch.Tensor): The output tensor to be padded. - drop_last_samples (bool, optional): Whether to drop the last samples during gathering. - Last samples are dropped when the number of samples is not divisible by the number of processes. - Defaults to True. - num_samples (int, optional): Number of samples per batch item. - - Returns: - torch.Tensor: The padded output tensor and the gathered length tensor. - """ - # Create a tensor of size batch_size, [output_length] * batch_size, for each process - # output_tensor can be of size: batch_size * num_samples * length_item or just batch_size * length_item - length_tensor = torch.tensor([output_tensor.shape[-1]] * output_tensor.shape[0], device=self.device) - if self.accelerator is not None: - # Gather all the lengths, we end up with a tensor of size num_processes [output_length_1, output_length_2, ...] - length_tensor = self.accelerator.gather(length_tensor) - # We pad the output_tensor to the max length - max_length = length_tensor.max().item() - padding = ( - (0, max_length - output_tensor.shape[-1], 0, 0, 0, 0) - if num_samples is not None - else (0, max_length - output_tensor.shape[-1], 0, 0) - ) - output_tensor = F.pad(output_tensor, padding, value=self.tokenizer.pad_token_id) - if self.accelerator: - if drop_last_samples: - output_tensor = self.accelerator.gather_for_metrics(output_tensor) - else: - output_tensor = self.accelerator.gather(output_tensor) - return output_tensor, length_tensor - - def tok_decode(self, tokens: torch.LongTensor) -> list[str]: - return self.tokenizer.batch_decode(tokens, skip_special_tokens=self.skip_special_tokens) - - -class MultiTokenEOSCriteria(transformers.StoppingCriteria): - """Criteria to stop on the specified multi-token sequence.""" - - def __init__( - self, - sequence: str, - tokenizer: transformers.PreTrainedTokenizer, - batch: Batch = None, - input_ids_shape: Tuple[int, int] = None, - ): - if batch is not None: - initial_decoder_input_length = batch.input_ids.shape[1] - batch_size = batch.input_ids.shape[0] - else: - initial_decoder_input_length = input_ids_shape[1] - batch_size = input_ids_shape[0] - - self.initial_decoder_input_length = initial_decoder_input_length - self.done_tracker = [False] * batch_size - self.sequence = sequence - self.sequence_ids = tokenizer.encode(sequence, add_special_tokens=False) - self.sequence_id_len = len(self.sequence_ids) - self.tokenizer = tokenizer - - def __call__(self, input_ids, scores, **kwargs) -> bool: - # For efficiency, we compare the last n tokens where n is the number of tokens in the stop_sequence - lookback_ids_batch = input_ids[:, self.initial_decoder_input_length :][:, -self.sequence_id_len :] - - lookback_tokens_batch = self.tokenizer.batch_decode(lookback_ids_batch) - - for i, done in enumerate(self.done_tracker): - if not done: - self.done_tracker[i] = self.sequence in lookback_tokens_batch[i] - return False not in self.done_tracker - - -def stop_sequences_criteria( - tokenizer: transformers.PreTrainedTokenizer, - stop_sequences: list[str], - batch: Batch, -) -> transformers.StoppingCriteriaList: - return transformers.StoppingCriteriaList( - [ - *[MultiTokenEOSCriteria(sequence, tokenizer, batch) for sequence in stop_sequences], - ] - ) diff --git a/src/lighteval/models/transformers/vlm_transformers_model.py b/src/lighteval/models/transformers/vlm_transformers_model.py deleted file mode 100644 index 0697ab729..000000000 --- a/src/lighteval/models/transformers/vlm_transformers_model.py +++ /dev/null @@ -1,443 +0,0 @@ -# MIT License - -# Copyright (c) 2025 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import logging -import os -from datetime import timedelta -from typing import Optional, Tuple, Union - -import torch -from accelerate import Accelerator, InitProcessGroupKwargs -from accelerate.utils import gather_object, get_max_memory -from pydantic import PositiveInt -from torch.utils.data import DataLoader -from tqdm import tqdm -from transformers import ( - AutoConfig, - AutoModelForImageTextToText, - AutoProcessor, -) -from transformers.configuration_utils import PretrainedConfig -from transformers.utils.quantization_config import BitsAndBytesConfig - -from lighteval.data import GenerativeTaskDataset -from lighteval.models.abstract_model import LightevalModel, ModelConfig -from lighteval.models.model_output import ModelResponse -from lighteval.models.utils import _get_dtype, _get_model_sha, _simplify_name -from lighteval.tasks.prompt_manager import PromptManager -from lighteval.tasks.requests import Doc, SamplingMethod -from lighteval.utils.cache_management import SampleCache, cached -from lighteval.utils.imports import ( - is_package_available, -) - - -logger = logging.getLogger(__name__) - - -class BatchCollator: - """Collator for batching requests""" - - def __init__(self, prompt_manager, processor, **kwargs): - self.processor = processor - self.prompt_manager = prompt_manager - self.kwargs = kwargs - - def __call__(self, requests: list[Doc]) -> Tuple[dict[str, torch.Tensor], list[Doc], list[str]]: - texts = [self.prompt_manager.prepare_prompt_multimodal(request) for request in requests] - images = [request.images for request in requests] - inputs = self.processor(text=texts, images=images, **self.kwargs) - return inputs, requests, texts - - -class VLMTransformersModelConfig(ModelConfig): - """Configuration class for VLM (image-text-to-text) models. - - Attributes: - model_name (str): - HuggingFace Hub model ID name or the path to a pre-trained - model to load. This is effectively the `pretrained_model_name_or_path` - argument of `from_pretrained` in the HuggingFace `transformers` API. - processor (Optional[str]): HuggingFace Hub processor ID that will be - used for preprocessing images and text. - use_fast_image_processor (Optional[bool]): - Whether to use a fast image processor. Not all VLMs support this yet. - subfolder (Optional[str]): The subfolder within the model repository. - revision (str): The revision of the model. - batch_size (int): The batch size for model training. - generation_size (Optional[int]): The maximum number of tokens to generate. - max_length (Optional[int]): The maximum length of the input + generated output. - add_special_tokens (bool, optional, defaults to True): Whether to add special tokens to the input sequences. - model_parallel (bool, optional, defaults to None): - True/False: force to use or not the `accelerate` library to load a large - model across multiple devices. - Default: None which corresponds to comparing the number of processes with - the number of GPUs. If it's smaller => model-parallelism, else not. - dtype (Union[str, torch.dtype], optional, defaults to None): - Converts the model weights to `dtype`, if specified. Strings get - converted to `torch.dtype` objects (e.g. `float16` -> `torch.float16`). - Use `dtype="auto"` to derive the type from the model's weights. - device (Union[int, str]): device to use for model training. - quantization_config (Optional[BitsAndBytesConfig]): quantization - configuration for the model, manually provided to load a normally floating point - model at a quantized precision. Needed for 4-bit and 8-bit precision. - trust_remote_code (bool): Whether to trust remote code during model - loading. - compile (bool, optional, defaults to False): Whether to compile the model for faster inference. - device_map (str | None, optional, defaults to None): Device mapping strategy for model loading. - generation_parameters (GenerationParameters, optional, defaults to empty GenerationParameters): - Configuration parameters that control text generation behavior, including - temperature, top_p, max_new_tokens, etc. - system_prompt (str | None, optional, defaults to None): Optional system prompt to be used with chat models. - This prompt sets the behavior and context for the model during evaluation. - cache_dir (str, optional, defaults to "~/.cache/huggingface/lighteval"): Directory to cache the model. - """ - - model_name: str - processor: str | None = None - use_fast_image_processor: bool | None = None - subfolder: str | None = None - revision: str = "main" - batch_size: PositiveInt = 1 - generation_size: PositiveInt | None = None - max_length: PositiveInt | None = None - add_special_tokens: bool = True - model_parallel: bool | None = None - dtype: str | None = None - device: Union[int, str] = "cuda" - trust_remote_code: bool = False - compile: bool = False - device_map: str | None = None - - def get_model_sha(self): - return _get_model_sha(repo_id=self.model_name, revision=self.revision) - - def get_transformers_config(self) -> PretrainedConfig: - revision = f"{self.revision}/{self.subfolder}" if self.subfolder else self.revision - config = AutoConfig.from_pretrained( - self.model_name, - revision=revision, - trust_remote_code=self.trust_remote_code, - ) - return config - - -class VLMTransformersModel(LightevalModel): - def __init__( - self, - config: VLMTransformersModelConfig, - ): - """Initializes a HuggingFace `AutoModel` and `AutoTokenizer` for evaluation.""" - self.accelerator = Accelerator(kwargs_handlers=[InitProcessGroupKwargs(timeout=timedelta(seconds=3000))]) - self.device = self.accelerator.device - self.torch_dtype = _get_dtype(config.dtype) - - # Config attributes - self.config = config - self.use_chat_template = True - self.batch_size = config.batch_size - - # Model, config, and processor - self.model_sha = config.get_model_sha() - self.model_name = _simplify_name(config.model_name) - self.model = self._create_auto_model() - self.processor = self._create_auto_processor() - self.transformers_config = config.get_transformers_config() - - # Attributes exposed by @property - self._max_length = self._init_max_length() - self._add_special_tokens = config.add_special_tokens or False - - # Generation config - self.generation_config_dict = config.generation_parameters.to_transformers_dict() - self.generation_config_dict["pad_token_id"] = self.pad_token_id - self.generation_config_dict["eos_token_id"] = self.eos_token_id - self.generation_config_dict["renormalize_logits"] = True - - self.prompt_manager = PromptManager( - use_chat_template=True, tokenizer=self.tokenizer, system_prompt=config.system_prompt - ) - - # Initialize cache for tokenization and predictions - self._cache = SampleCache(config) - - @property - def tokenizer(self): - return self.processor.tokenizer - - @property - def pad_token_id(self): - return self.tokenizer.pad_token_id if self.tokenizer.pad_token_id else self.tokenizer.eos_token_id - - @property - def eos_token_id(self): - return self.tokenizer.eos_token_id - - @property - def add_special_tokens(self): - return self._add_special_tokens - - @property - def max_length(self): - return self._max_length - - @property - def disable_tqdm(self) -> bool: - disable_tqdm = False - if self.accelerator: - disable_tqdm = bool(not self.accelerator.is_main_process) - return disable_tqdm - - # Copied from ./transformers_model.py - def init_model_parallel(self, model_parallel: bool | None = None) -> Tuple[bool, Optional[dict], Optional[str]]: - """Compute all the parameters related to model_parallel""" - if not is_package_available("accelerate"): - return False, None, None - - self.num_local_processes = int(os.environ.get("LOCAL_WORLD_SIZE", 1)) - self.num_machines = torch.cuda.device_count() // self.num_local_processes - - if self.num_machines == 1: - logger.info("We are not in a distributed setting. Setting model_parallel to False.") - model_parallel = False - - if model_parallel is None: - max_memory_all_gpus = get_max_memory() # A dict of the max memory for all the gpus - if "cpu" in max_memory_all_gpus: - del max_memory_all_gpus["cpu"] - model_parallel = bool(self.num_local_processes < len(max_memory_all_gpus)) - logger.info( - f"Setting model parallel to {model_parallel} since " - f"the number of local processes is {self.num_local_processes} " - f"and the number of GPUs is {len(max_memory_all_gpus)}" - ) - if model_parallel is True: - max_memory_all_gpus = get_max_memory() # A dict of the max memory for all the gpus - if "cpu" in max_memory_all_gpus: - del max_memory_all_gpus["cpu"] - max_mem_this_process = { - k: v - for k, v in max_memory_all_gpus.items() - if k % self.num_local_processes == (self.accelerator.process_index % self.num_local_processes) - } - device_map = "auto" - logger.info( - f"Model parallel was set to True, setting max memory per GPU to {max_mem_this_process} and device map to {device_map}" - ) - else: - max_mem_this_process = None - device_map = None - logger.info( - f"Model parallel was set to False, max memory set to {max_mem_this_process} and device map to {device_map}" - ) - return model_parallel, max_mem_this_process, device_map - - @staticmethod - def _get_quantization_config(config: VLMTransformersModelConfig) -> BitsAndBytesConfig | None: - if config.dtype == "4bit": - quantization_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.float16) - elif config.dtype == "8bit": - quantization_config = BitsAndBytesConfig(load_in_8bit=True) - else: - quantization_config = None - return quantization_config - - def _create_auto_model(self): - model_parallel, max_memory, device_map = self.init_model_parallel(self.config.model_parallel) - self.config.model_parallel = model_parallel - - quantization_config = self._get_quantization_config(self.config) - - subfolder = self.config.subfolder - revision = f"{self.config.revision}/{subfolder}" if subfolder is not None else self.config.revision - - model = AutoModelForImageTextToText.from_pretrained( - self.config.model_name, - revision=revision, - device_map=device_map, - max_memory=max_memory, - torch_dtype=self.torch_dtype, - quantization_config=quantization_config, - trust_remote_code=self.config.trust_remote_code, - ) - model.eval() - torch.set_grad_enabled(False) - - if self.config.compile: - raise NotImplementedError("Compiling VLM models is not supported yet") - - # We are in DP (and launch the script with `accelerate launch`) - if model_parallel is False and self.config.dtype not in ["4bit", "8bit"]: - logger.info(f"Using Data Parallelism, putting model on device {self.device}") - model = model.to(self.device) - - return model - - def _create_auto_processor(self): - """Create a transformers `Processor` for VLM (image-text-to-text) model. - - Returns: - transformers.ProcessorMixin: The created processor. - """ - processor_name = self.config.processor or self.config.model_name - revision, subfolder = self.config.revision, self.config.subfolder - revision = revision if not subfolder else f"{revision}/{subfolder}" - - processor = AutoProcessor.from_pretrained( - processor_name, - revision=revision, - padding_side="left", - truncation_side="left", - trust_remote_code=self.config.trust_remote_code, - use_fast=self.config.use_fast_image_processor, - ) - - return processor - - def _init_max_length(self) -> int: - """Return the maximum sequence length of the model. - - NOTE: For relative position encoded models you should specify the max - sequence length of the model in the constructor via `max_length`. - - Returns: - int: Max length to use depending on the available args and config - """ - if self.config.max_length is not None: - return self.config.max_length - - # Try to get the sequence length from the model config. It's no super robust - text_model_config = self.transformers_config.get_text_config() - max_seq_length = getattr(text_model_config, "max_position_embeddings", None) - if max_seq_length is not None: - return max_seq_length - - logger.warning( - "No max_length attribute found in the model config. Using the default max sequence length setting `2048`. " - "It is recommended to set max_length trough the model args: max_length=..." - ) - - return 2048 - - @cached(SamplingMethod.GENERATIVE) - def greedy_until( - self, - docs: list[Doc], - ) -> list[ModelResponse]: - """Generates responses using a greedy decoding strategy until certain ending conditions are met. - - Args: - docs (list[Docs]): list of docs containing the context and ending conditions. - - Returns: - list[ModelResponse]: list of generated responses. - """ - return self._greedy_until(docs) - - def _greedy_until( - self, - docs: list[Doc], - ) -> list[ModelResponse]: - """Wrapper for the greedy until logic, to avoid interface changes in the future""" - # Tokenizing context for sorting in the dataset - - dataset = GenerativeTaskDataset(requests=docs, num_dataset_splits=self.DATASET_SPLITS) - collator = BatchCollator( - self.prompt_manager, - self.processor, - truncation="longest_first", # we truncate to the model max length if needed - padding="longest", # we pad to the longest sequence - max_length=self.max_length - 1, # we should always allow minimum one token of generation - add_special_tokens=self.add_special_tokens, - return_tensors="pt", - ) - - results = [] - for split in dataset.splits_iterator(): - batch_size = self.batch_size or 1 - dataloader = DataLoader(split, batch_size=batch_size, collate_fn=collator) - if self.accelerator: - dataloader = self.accelerator.prepare(dataloader) - - for batch_inputs, batch_requests, input_context in tqdm( - dataloader, desc="Greedy generation", position=1, leave=True, disable=self.disable_tqdm - ): - batch_inputs = batch_inputs.to(self.device) - if self.torch_dtype is not None: - batch_inputs = batch_inputs.to(self.torch_dtype) - - max_new_tokens = self.config.generation_size or batch_requests[0].generation_size - num_samples = batch_requests[0].num_samples - do_sample = num_samples > 1 or self.generation_config_dict["temperature"] > 0 - - if num_samples > 1 and self.generation_config_dict["temperature"] == 0: - raise ValueError( - "num_samples > 1 is not supported with temperature=0, please set temperature > 0 or use non sampling metrics." - ) - - outputs = self.model.generate( - **batch_inputs, - **self.generation_config_dict, # custom generation params - max_new_tokens=max_new_tokens, - num_return_sequences=batch_requests[0].num_samples, - output_logits=batch_requests[0].use_logits, - do_sample=do_sample, - ) - input_tokens = batch_inputs.input_ids - generated_tokens = outputs.sequences[:, input_tokens.shape[1] :] - generated_texts = self.processor.batch_decode(generated_tokens, skip_special_tokens=True) - attention_mask = batch_inputs["attention_mask"] - padded_tokens_count = (attention_mask == 0).sum(dim=1) - - batch_results = [] - for i in range(len(generated_texts)): - generated_response = ModelResponse( - input=input_context[i], - text=generated_texts[i], - output_tokens=generated_tokens[i].cpu().numpy(), - input_tokens=input_tokens[i].cpu().numpy(), - truncated_tokens_count=-1, - padded_tokens_count=padded_tokens_count[i].item(), - logits=outputs.logits[i].cpu().numpy() if outputs.logits is not None else None, - ) - batch_results.append(generated_response) - - if self.accelerator: - batch_results = gather_object(batch_results) - - results.extend(batch_results) - - return dataset.get_original_order(results) - - @cached(SamplingMethod.LOGPROBS) - def loglikelihood( - self, - docs: list[Doc], - ) -> list[ModelResponse]: - raise NotImplementedError() - - @cached(SamplingMethod.PERPLEXITY) - def loglikelihood_rolling( - self, - docs: list[Doc], - ) -> list[ModelResponse]: - raise NotImplementedError() diff --git a/src/lighteval/models/utils.py b/src/lighteval/models/utils.py deleted file mode 100644 index f615019ea..000000000 --- a/src/lighteval/models/utils.py +++ /dev/null @@ -1,137 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import logging -import os -from itertools import islice -from typing import Optional, Union - -import torch -from huggingface_hub import HfApi -from transformers import AutoTokenizer -from transformers.models.auto.configuration_auto import AutoConfig - - -logger = logging.getLogger(__name__) - - -def _get_dtype(dtype: Union[str, torch.dtype, None], config: Optional[AutoConfig] = None) -> Optional[torch.dtype]: - """Get the torch dtype based on the input arguments. - - Args: - dtype (Union[str, torch.dtype]): The desired dtype. Can be a string or a torch dtype. - config (Optional[transformers.AutoConfig]): The model config object. Defaults to None. - - Returns: - torch.dtype: The torch dtype based on the input arguments. - """ - if config is not None and hasattr(config, "quantization_config"): - # must be infered - return None - - if dtype is not None: - if isinstance(dtype, str) and dtype not in ["auto", "4bit", "8bit"]: - # Convert `str` args torch dtype: `float16` -> `torch.float16` - return getattr(torch, dtype) - elif isinstance(dtype, torch.dtype): - return dtype - - if config is not None: - return config.torch_dtype - - return None - - -def _simplify_name(name_or_path: str) -> str: - """If the model is loaded from disk, then the name will have the following format: - /p/a/t/h/models--org--model_name/revision/model_files - This function return the model_name as if it was loaded from the hub: - org/model_name - - Args: - name_or_path (str): The name or path to be simplified. - - Returns: - str: The simplified name. - """ - if os.path.isdir(name_or_path) or os.path.isfile(name_or_path): # Loading from disk - simple_name_list = name_or_path.split("/") - # The following manages files stored on disk, loaded with the hub model format: - # /p/a/t/h/models--org--model_name/revision/model_files - if any("models--" in item for item in simple_name_list): # Hub format - simple_name = [item for item in simple_name_list if "models--" in item][0] - simple_name = simple_name.replace("models--", "").replace("--", "/") - return simple_name - # This is for custom folders - else: # Just a folder, we don't know the shape - return name_or_path.replace("/", "_") - - return name_or_path - - -def _get_model_sha(repo_id: str, revision: str): - api = HfApi() - try: - model_info = api.model_info(repo_id=repo_id, revision=revision) - return model_info.sha - except Exception: - return "" - - -def batched(iterable, n): - # batched('ABCDEFG', 3) → ABC DEF G - if n < 1: - raise ValueError("n must be at least one") - it = iter(iterable) - while batch := tuple(islice(it, n)): - yield batch - - -def uses_chat_template( - model_name: str = None, tokenizer: AutoTokenizer = None, override_chat_template: bool = None -) -> bool: - """Returns a boolean depending on whether the Transformers AutoTokenizer contains - a chat template or not - - Args: - model_name (str, optional): Model name on HF. - tokenizer (AutoTokenizer, optional): The tokenizer to check. - override_chat_template (bool, optional): Override the chat template detection. - - Returns: - bool: True if Tokenizer config contains a chat template, False otherwise - """ - if override_chat_template is not None: - return override_chat_template - if model_name is None and tokenizer is None: - raise Exception("`uses_chat_template` requires either a tokenizer or model name as input") - try: - if tokenizer: - tk = tokenizer - else: - tk = AutoTokenizer.from_pretrained(model_name) - return tk.chat_template is not None - except Exception: - logger.warning( - "We were not able to detect if the chat template should be used for your model: {e}. Assuming we're using a chat template" - ) - return True diff --git a/src/lighteval/models/vllm/vllm_model.py b/src/lighteval/models/vllm/vllm_model.py deleted file mode 100644 index 969caf8fa..000000000 --- a/src/lighteval/models/vllm/vllm_model.py +++ /dev/null @@ -1,696 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import asyncio -import gc -import itertools -import logging -import os -from typing import Coroutine, Optional - -import torch -from pydantic import NonNegativeFloat, NonNegativeInt, PositiveInt -from tqdm import tqdm - -from lighteval.data import GenerativeTaskDataset, LoglikelihoodDataset -from lighteval.models.abstract_model import LightevalModel, ModelConfig -from lighteval.models.model_output import ModelResponse -from lighteval.models.utils import _simplify_name, uses_chat_template -from lighteval.tasks.prompt_manager import PromptManager -from lighteval.tasks.requests import Doc, SamplingMethod -from lighteval.utils.cache_management import SampleCache, cached -from lighteval.utils.imports import is_package_available, requires - - -logger = logging.getLogger(__name__) - - -if is_package_available("vllm"): - import ray - from more_itertools import distribute - from vllm import LLM, RequestOutput, SamplingParams - from vllm.distributed.parallel_state import ( - destroy_distributed_environment, - destroy_model_parallel, - ) - from vllm.transformers_utils.tokenizer import get_tokenizer - from vllm.v1.engine.async_llm import AsyncEngineArgs, AsyncLLM - - logging.getLogger("vllm").propagate = True - logging.getLogger("vllm").handlers.clear() - - logging.getLogger("ray").propagate = True - logging.getLogger("ray").handlers.clear() -else: - from unittest.mock import Mock - - LLM = SamplingParams = get_tokenizer = ray = distribute = destroy_distributed_environment = ( - destroy_model_parallel - ) = Mock() - AsyncLLM = AsyncEngineArgs = RequestOutput = Mock() - -os.environ["TOKENIZERS_PARALLELISM"] = "false" - -STARTING_BATCH_SIZE = 512 - - -class VLLMModelConfig(ModelConfig): - """Configuration class for VLLM inference engine. - - This configuration is used to load and configure models using the VLLM inference engine, - which provides high-performance inference for large language models with features like - PagedAttention, continuous batching, and efficient memory management. - - vllm doc: https://docs.vllm.ai/en/v0.7.1/serving/engine_args.html - - Attributes: - model_name (str): - HuggingFace Hub model ID or path to the model to load. - tokenizer (str | None): - HuggingFace Hub model ID or path to the tokenizer to load. - revision (str): - Git revision of the model. Defaults to "main". - dtype (str): - Data type for model weights. Defaults to "bfloat16". Options: "float16", "bfloat16", "float32". - tensor_parallel_size (PositiveInt): - Number of GPUs to use for tensor parallelism. Defaults to 1. - data_parallel_size (PositiveInt): - Number of GPUs to use for data parallelism. Defaults to 1. - pipeline_parallel_size (PositiveInt): - Number of GPUs to use for pipeline parallelism. Defaults to 1. - gpu_memory_utilization (NonNegativeFloat): - Fraction of GPU memory to use. Lower this if running out of memory. Defaults to 0.9. - enable_prefix_caching (bool): - Whether to enable prefix caching to speed up generation. May use more memory. Should be disabled for LFM2. Defaults to True. - max_model_length (PositiveInt | None): - Maximum sequence length for the model. If None, automatically inferred. - Reduce this if encountering OOM issues (4096 is usually sufficient). - quantization (str | None): - Quantization method. - load_format (str | None): - The format of the model weights to load. choices: auto, pt, safetensors, npcache, dummy, tensorizer, sharded_state, gguf, bitsandbytes, mistral, runai_streamer. - swap_space (PositiveInt): - CPU swap space size in GiB per GPU. Defaults to 4. - seed (NonNegativeInt): - Random seed for reproducibility. Defaults to 1234. - trust_remote_code (bool): - Whether to trust remote code when loading models. Defaults to False. - add_special_tokens (bool): - Whether to add special tokens during tokenization. Defaults to True. - multichoice_continuations_start_space (bool): - Whether to add a space before multiple choice continuations. Defaults to True. - pairwise_tokenization (bool): - Whether to tokenize context and continuation separately for loglikelihood evals. Defaults to False. - max_num_seqs (PositiveInt): - Maximum number of sequences per iteration. Controls batch size at prefill stage. Defaults to 128. - max_num_batched_tokens (PositiveInt): - Maximum number of tokens per batch. Defaults to 2048. - subfolder (str | None): - Subfolder within the model repository. Defaults to None. - is_async (bool): - Whether to use the async version of VLLM. Defaults to False. - override_chat_template (bool): - If True, we force the model to use a chat template. If alse, we prevent the model from using - a chat template. If None, we use the default (true if present in the tokenizer, false otherwise) - generation_parameters (GenerationParameters, optional, defaults to empty GenerationParameters): - Configuration parameters that control text generation behavior, including - temperature, top_p, max_new_tokens, etc. - system_prompt (str | None, optional, defaults to None): Optional system prompt to be used with chat models. - This prompt sets the behavior and context for the model during evaluation. - cache_dir (str, optional, defaults to "~/.cache/huggingface/lighteval"): Directory to cache the model. - - Example: - ```python - config = VLLMModelConfig( - model_name="meta-llama/Llama-3.1-8B-Instruct", - tensor_parallel_size=2, - gpu_memory_utilization=0.8, - max_model_length=4096, - generation_parameters=GenerationParameters( - temperature=0.7, - max_new_tokens=100 - ) - ) - ``` - """ - - model_name: str - tokenizer: str | None = None - revision: str = "main" # revision of the model - dtype: str = "bfloat16" - tensor_parallel_size: PositiveInt = 1 # how many GPUs to use for tensor parallelism - data_parallel_size: PositiveInt = 1 # how many GPUs to use for data parallelism - pipeline_parallel_size: PositiveInt = 1 # how many GPUs to use for pipeline parallelism - gpu_memory_utilization: NonNegativeFloat = 0.9 # lower this if you are running out of memory - enable_prefix_caching: bool = None # whether to enable prefix caching to speed up generation. May use more memory. Should be disabled for LFM2 - max_model_length: PositiveInt | None = ( - None # maximum length of the model, ussually infered automatically. reduce this if you encouter OOM issues, 4096 is usually enough - ) - quantization: str | None = None - load_format: str | None = None - swap_space: PositiveInt = 4 # CPU swap space size (GiB) per GPU. - seed: NonNegativeInt = 1234 - trust_remote_code: bool = False - add_special_tokens: bool = True - multichoice_continuations_start_space: bool = ( - True # whether to add a space at the start of each continuation in multichoice generation - ) - pairwise_tokenization: bool = False # whether to tokenize the context and continuation separately or together. - max_num_seqs: PositiveInt = 128 # maximum number of sequences per iteration; This variable and `max_num_batched_tokens` effectively control the batch size at prefill stage. See https://github.com/vllm-project/vllm/issues/2492 for detailed explaination. - max_num_batched_tokens: PositiveInt = 2048 # maximum number of tokens per batch - subfolder: str | None = None - is_async: bool = False # Whether to use the async version or sync version of the model - override_chat_template: bool = None - - -@requires("vllm") -class VLLMModel(LightevalModel): - def __init__( - self, - config: VLLMModelConfig, - ): - """Initializes a HuggingFace `AutoModel` and `AutoTokenizer` for evaluation.""" - self.config = config - self.use_chat_template = uses_chat_template( - model_name=config.model_name, override_chat_template=config.override_chat_template - ) - self.data_parallel_size = config.data_parallel_size - self.tensor_parallel_size = config.tensor_parallel_size - self._add_special_tokens = config.add_special_tokens if config.add_special_tokens is not None else False - self._tokenizer = self._create_auto_tokenizer(config) - - self._max_length = ( - config.max_model_length - ) # will be None if the config is None, then defined in _create_auto_model - - # If model_parallel is not set we compare the number of processes with the number of GPUs - self.model = self._create_auto_model(config) - - # self._device = config.accelerator.device if config.accelerator is not None else "cpu" - self.multichoice_continuations_start_space = config.multichoice_continuations_start_space - - self.model_name = _simplify_name(config.model_name) - self.model_sha = "" - self.precision = config.dtype - - self.pairwise_tokenization = config.pairwise_tokenization - - self.prompt_manager = PromptManager(self.use_chat_template, self.tokenizer, config.system_prompt) - - # Initialize cache for tokenization and predictions - self._cache = SampleCache(config) - - @property - def tokenizer(self): - return self._tokenizer - - def cleanup(self): - destroy_model_parallel() - if self.model is not None: - del self.model - gc.collect() - ray.shutdown() - destroy_distributed_environment() - torch.cuda.empty_cache() - - @property - def add_special_tokens(self): - return self._add_special_tokens - - @property - def max_length(self) -> int: - return self._max_length - - def _create_auto_model(self, config: VLLMModelConfig) -> Optional[LLM]: - """Creates an instance of the pretrained HF model. - - Args: - config (VLLMModelConfig): The VLLM model configuration. - - Returns: - Optional[LLM]: The created auto model instance. - """ - self.model_args = { - "model": config.model_name, - "gpu_memory_utilization": config.gpu_memory_utilization, - "enable_prefix_caching": config.enable_prefix_caching, - "revision": config.revision + (f"/{config.subfolder}" if config.subfolder is not None else ""), - "dtype": config.dtype, - "trust_remote_code": config.trust_remote_code, - "tensor_parallel_size": config.tensor_parallel_size, - "pipeline_parallel_size": config.pipeline_parallel_size, - "max_model_len": self._max_length, - "swap_space": 4, - "seed": int(config.seed), - "max_num_seqs": int(config.max_num_seqs), - "max_num_batched_tokens": int(config.max_num_batched_tokens), - "enforce_eager": True, - } - - if config.quantization is not None: - self.model_args["quantization"] = config.quantization - if config.load_format is not None: - self.model_args["load_format"] = config.load_format - - if config.data_parallel_size > 1: - self.model_args["distributed_executor_backend"] = "ray" - self._batch_size = "auto" - - if self._max_length is None: - # Todo: we will want to manage this automatically - atm this arg must be set at least 2 times (in gen params + model args) for - # vllm models, which is an issue. - logger.warning( - "The model max_length was not set in the model arguments. Since the model is using data parallelism, it is created later " - " with `ray`, so we can't infer the max_length automatically atm. It might raise issues later on: if it does, relaunch your " - "run, but set `max_model_length` explicitely in the model args." - ) - return None - - model = LLM(**self.model_args) - - # If the max_length can't get extracted from the config, it will be inferred from the model - # Inferring from the tokenizer will cause vllm to bug for models with mismatches between model - # config and tk config, like mistralai/Mistral-7B-v0.1 - if self._max_length is None: - self._max_length = model.llm_engine.model_config.max_seq_len_to_capture - - return model - - def _create_auto_tokenizer(self, config: VLLMModelConfig): - tokenizer = get_tokenizer( - config.tokenizer or config.model_name, # use HF tokenizer for non-HF models, like GGUF model. - tokenizer_mode="auto", - trust_remote_code=config.trust_remote_code, - revision=config.revision, - ) - tokenizer.pad_token = tokenizer.eos_token - return tokenizer - - @cached(SamplingMethod.GENERATIVE) - def greedy_until( - self, - docs: list[Doc], - ) -> list[ModelResponse]: - """Generates responses using a greedy decoding strategy until certain ending conditions are met. - - Args: - docs (list[Doc]): List of documents containing the context for generation. - - Returns: - list[ModelResponse]: list of generated responses. - """ - return self._greedy_until(docs) - - def _greedy_until( - self, - docs: list[Doc], - ) -> list[ModelResponse]: - dataset = GenerativeTaskDataset(requests=docs, num_dataset_splits=self.DATASET_SPLITS) - results = [] - - for split in tqdm( - dataset.splits_iterator(), - total=dataset.num_dataset_splits, - desc="Splits", - position=0, - disable=False, # self.disable_tqdm, - ): - # For chat models, generation stops with EOS token, so we don't need to specify stop tokens - if self.use_chat_template: - stop_tokens = [] - else: - # NOTE: we are assuming all items in a batch behave similarly (same - # stop_tokens and max_tokens genrated) which is not necessarily - # the case! Because of that we only use batch size of 1 - stop_tokens = split[0].stop_sequences or [] - - max_new_tokens = self.config.generation_parameters.max_new_tokens or split[0].generation_size - num_samples = split[0].num_samples - - context = [self.prompt_manager.prepare_prompt(doc) for doc in split] - tokenized = self.tokenizer(context, add_special_tokens=self.add_special_tokens) - - # The main question for this step is the following: - # Would we rather truncate the prompt to allow generation to go to max_new_tokens, at the risk - # of losing some meaning, or have some generations that are exceedingly short? - # The choice we go for here is to avoid truncating the prompt if we can, since it - # should have been managed by the prompt creator/few shot manager if requested by the user. - inputs = tokenized["input_ids"] - context_size = len(inputs[0]) - - # left truncate the inputs to the maximum length - if self.max_length is None: - logger.warning( - "The model max_length was not set in the model arguments, so we cannot check if we need to truncate the context." - ) - elif max_new_tokens is not None: - if context_size + max_new_tokens > self.max_length: - logger.warning( - f"{context_size + max_new_tokens=} which is greater than {self.max_length=}. Truncating context to {self.max_length - max_new_tokens} tokens." - ) - context_size = self.max_length - max_new_tokens - if context_size < 0: - logger.critical( - f"{context_size=} is less than 0, either reduce the max_new_tokens or increase model max length." - ) - raise ValueError("Context size is less than 0.") - inputs = [input[-context_size:] for input in inputs] - else: - if context_size > self.max_length: - logger.warning( - f"{context_size=} which is greater than {self.max_length=}. Truncating context to {self.max_length} tokens." - ) - context_size = self.max_length - inputs = [input[-context_size:] for input in inputs] - - vllm_outputs = self._generate( - inputs=inputs, - max_new_tokens=max_new_tokens, - stop_tokens=stop_tokens, - returns_logits=False, - num_samples=num_samples, - ) - - for i, vllm_output in enumerate(vllm_outputs): - output_token_ids = [outputs.token_ids for outputs in vllm_output.outputs] - result = [output.text for output in vllm_output.outputs] - input_token_ids = vllm_output.prompt_token_ids - - cur_response = ModelResponse( - input=context[i], - text=result, - output_tokens=list(output_token_ids), - input_tokens=input_token_ids, - ) - results.append(cur_response) - - return dataset.get_original_order(results) - - def _generate( - self, - inputs: list[list[int]], - max_new_tokens: Optional[int] = None, - stop_tokens: Optional[list[str]] = None, - returns_logits: Optional[bool] = False, - num_samples: int = 1, - generate: bool = True, - ) -> list: - """Contains the actual logic of the generation.""" - sampling_params = SamplingParams(**self.config.generation_parameters.to_vllm_dict()) - - if generate: - sampling_params.n = num_samples - sampling_params.max_tokens = max_new_tokens - sampling_params.stop = stop_tokens - sampling_params.logprobs = 1 if returns_logits else 0 - if num_samples > 1 and sampling_params.temperature == 0: - raise ValueError( - "num_samples > 1 is not supported with temperature=0, please set temperature > 0 or use non sampling metrics." - ) - else: - sampling_params.temperature = 0 - sampling_params.prompt_logprobs = 1 - sampling_params.max_tokens = 1 - sampling_params.detokenize = False - - if self.data_parallel_size > 1: - - @ray.remote(num_gpus=self.tensor_parallel_size) - def run_inference_one_model(model_args: dict, sampling_params: SamplingParams, requests): - llm = LLM(**model_args) - return llm.generate(prompt_token_ids=requests, sampling_params=sampling_params) - - # dispatch requests to all self.data_parallel_size workers, in interleaved fashion - # interleaved important to balance context lengths across workers - requests = [list(x) for x in distribute(self.data_parallel_size, inputs)] - inputs = ((self.model_args, sampling_params, req) for req in requests) - object_refs = [run_inference_one_model.remote(*x) for x in inputs] - results = ray.get(object_refs) - # Invoke ray.shutdown() to prevent hang-ups if subsequent calls required. - ray.shutdown() - # flatten results - outputs = [ - x - for x in itertools.chain.from_iterable(itertools.zip_longest(*[list(x) for x in results])) - if x is not None - ] - else: - outputs = self.model.generate( - prompt_token_ids=inputs, - sampling_params=sampling_params, - use_tqdm=True, - ) - - return outputs - - @cached(SamplingMethod.LOGPROBS) - def loglikelihood(self, docs: list[Doc]) -> list[ModelResponse]: - return self._loglikelihood_tokens(docs) - - def _loglikelihood_tokens( - self, - docs: list[Doc], - ) -> list[ModelResponse]: - dataset = LoglikelihoodDataset(requests=docs, num_dataset_splits=1) - res = [] - - for split in tqdm(dataset.splits_iterator()): - contexts = [self.prompt_manager.prepare_prompt(doc) for doc in split] - - inputs = [] - tokenized_continuations_batch = [] - tokenized_contexts_batch = [] - - for context, doc in zip(contexts, split): - tokenized_contexts, tokenized_continuations = self.tok_encode_pair( - context, doc.choices, pairwise=self.pairwise_tokenization - ) - for tokenized_context, tokenized_continuation in zip(tokenized_contexts, tokenized_continuations): - inputs.append(tokenized_context + tokenized_continuation) - tokenized_continuations_batch.append(tokenized_continuation) - tokenized_contexts_batch.append(tokenized_context) - - # Left truncate the inputs to the maximum length - if self.max_length: # can be None if the model is initialized with ray - inputs = [input[-self.max_length :] for input in inputs] - outputs = self._generate(inputs, generate=False) - - flat_index = 0 - for i, doc in enumerate(split): - outputs_doc = outputs[flat_index : flat_index + len(doc.choices)] - tokenized_continuations_doc = tokenized_continuations_batch[flat_index : flat_index + len(doc.choices)] - tokenized_contexts_doc = tokenized_contexts_batch[flat_index : flat_index + len(doc.choices)] - logprobs_doc = [] - argmax_doc = [] - output_tokens_doc = [] - input_tokens_doc = [] - - for output, context, continuation in zip( - outputs_doc, tokenized_contexts_doc, tokenized_continuations_doc - ): - continuation_logprobs = [] - for token, logprobs in zip(continuation[::-1], output.prompt_logprobs[::-1]): - continuation_logprobs.append(logprobs[token]) - - bool_score = all(logprob.rank == 1 for logprob in continuation_logprobs) - continuation_logprobs = [logprob.logprob for logprob in continuation_logprobs] - continuation_logprobs = sum(continuation_logprobs) - logprobs_doc.append(continuation_logprobs) - argmax_doc.append(bool_score) - output_tokens_doc.append(continuation) - input_tokens_doc.append(context) - - answer = ModelResponse( - input=contexts[i], - input_tokens=input_tokens_doc, - output_tokens=output_tokens_doc, - logprobs=logprobs_doc, - argmax_logits_eq_gold=argmax_doc, - ) - res.append(answer) - flat_index += len(doc.choices) - - return dataset.get_original_order(res) - - @cached(SamplingMethod.PERPLEXITY) - def loglikelihood_rolling(self, docs: list[Doc]) -> list[ModelResponse]: - raise NotImplementedError() - - -@requires("vllm") -class AsyncVLLMModel(VLLMModel): - """VLLM models which deploy async natively (no ray). Supports DP and PP/TP but not batch size > 1""" - - DATASET_SPLITS = 1 - is_async = True - - def cleanup(self): - gc.collect() - destroy_distributed_environment() - torch.cuda.empty_cache() - - def _create_auto_model(self, config: VLLMModelConfig): - """Creates an instance of the async vllm model loaded from HF. Requires using the v1 of VLLM. - - Returns: - AsyncLLM: The created async VLLM instance - """ - self.model_args = { - "model": config.model_name, - "gpu_memory_utilization": config.gpu_memory_utilization, - "revision": config.revision + (f"/{config.subfolder}" if config.subfolder is not None else ""), - "dtype": config.dtype, - "trust_remote_code": config.trust_remote_code, - "tensor_parallel_size": config.tensor_parallel_size, - "data_parallel_size": config.data_parallel_size, - "pipeline_parallel_size": config.pipeline_parallel_size, - "max_model_len": self._max_length, - "swap_space": 4, - "seed": int(config.seed), - "max_num_seqs": int(config.max_num_seqs), - "max_num_batched_tokens": int(config.max_num_batched_tokens), - "enforce_eager": True, - } - - if config.data_parallel_size > 1: - self._batch_size = "auto" - - model = AsyncLLM.from_engine_args(AsyncEngineArgs(**self.model_args)) - - # If the max_length can't get extracted from the config, it will be inferred from the model - if self._max_length is None: - self._max_length = model.model_config.max_seq_len_to_capture - - return model - - async def _async_one_item( - self, - index: int, - doc: Doc, - generative: bool, - ) -> Coroutine[None, list, str]: - """Contains the actual logic of the generation.""" - sampling_params = SamplingParams(**self.config.generation_parameters.to_vllm_dict()) - - if not generative: - sampling_params.temperature = 0 - sampling_params.prompt_logprobs = 1 - sampling_params.max_tokens = 1 - sampling_params.detokenize = False - prompt = self.prompt_manager.prepare_prompt(doc) + doc.choice - index_str = f"logprob_{index}" - else: - sampling_params.n = doc.num_samples - if sampling_params.n > 1: - # Todo clementine: investigate more - logger.warning( - "Careful, there can be unexpected behavior when using sampling evals with the async vllm model" - ) - sampling_params.max_tokens = self.config.generation_parameters.max_new_tokens or doc.generation_size - sampling_params.stop = [] if self.use_chat_template else doc.stop_sequences - sampling_params.logprobs = int(doc.use_logits) - prompt = self.prompt_manager.prepare_prompt(doc) - index_str = f"generative_{index}" - - generator = self.model.generate(request_id=index_str, prompt=prompt, sampling_params=sampling_params) - try: - while output := await anext(generator): - continue - except StopAsyncIteration: - pass - - return output - - async def _async_batch(self, docs: list[Doc], generative: bool) -> list: - processed_requests = [ - self._async_one_item(index=index, doc=doc, generative=generative) for index, doc in enumerate(docs) - ] - results = await asyncio.gather(*processed_requests) - return results - - @cached(SamplingMethod.GENERATIVE) - async def greedy_until( - self, - docs: list[Doc], - ) -> list[ModelResponse]: - """Generates responses using a greedy decoding strategy until certain ending conditions are met. - - Args: - docs (list[Doc]): List of documents containing the context for generation. - - Returns: - list[ModelResponse]: list of generated responses. - """ - results = [] - - responses = await self._async_batch(docs=docs, generative=True) - - for response in responses: - output_token_ids = [outputs.token_ids for outputs in response.outputs] - full_logprobs = [output.logprobs for output in response.outputs] or [] - logprobs = [logprob[token_id].logprob for token_id, logprob in zip(output_token_ids[0], full_logprobs[0])] - result = [output.text for output in response.outputs] - input_token_ids = response.prompt_token_ids - - cur_response = ModelResponse( - text=result, - logprobs=logprobs, - output_tokens=list(output_token_ids), - input_tokens=input_token_ids, - ) - results.append(cur_response) - - return results - - @cached(SamplingMethod.LOGPROBS) - async def loglikelihood( - self, - docs: list[Doc], - ) -> list[ModelResponse]: - """Generates responses using a greedy decoding strategy until certain ending conditions are met and - stores the logprobs. - - Args: - docs (list[Doc]): List of documents containing the context and choices. - - Returns: - list[ModelResponse]: list of generated responses. - """ - results = [] - - responses = await self._async_batch(docs=docs, generative=False) - - for response, input in zip(responses, docs): - continuation_logprobs = [] - for token, logprobs in zip(input.tokenized_continuation[::-1], response.prompt_logprobs[::-1]): - continuation_logprobs.append(logprobs[token]) - bool_score = all(logprob.rank == 1 for logprob in continuation_logprobs) - continuation_logprobs = [logprob.logprob for logprob in continuation_logprobs] - answer = ModelResponse( - input_tokens=input.tokenized_context + input.tokenized_continuation, - output_tokens=input.tokenized_continuation, - logprobs=sum(continuation_logprobs), - argmax_logits_eq_gold=bool_score, - ) - results.append(answer) - - return results diff --git a/src/lighteval/pipeline.py b/src/lighteval/pipeline.py deleted file mode 100644 index 1f5da9c14..000000000 --- a/src/lighteval/pipeline.py +++ /dev/null @@ -1,447 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import ast -import asyncio -import collections -import os -import random -from dataclasses import dataclass -from datetime import timedelta -from enum import Enum, auto - -import numpy as np -from tqdm import tqdm - -from lighteval.logging.evaluation_tracker import EvaluationTracker -from lighteval.metrics import apply_metric -from lighteval.models.abstract_model import LightevalModel, ModelConfig -from lighteval.models.model_loader import TransformersModel, load_model -from lighteval.models.model_output import ( - ModelResponse, -) -from lighteval.tasks.lighteval_task import LightevalTask -from lighteval.tasks.registry import Registry -from lighteval.tasks.requests import SamplingMethod -from lighteval.utils.imports import is_package_available -from lighteval.utils.parallelism import test_all_gather -from lighteval.utils.utils import make_results_table, remove_reasoning_tags - - -if is_package_available("accelerate"): - from accelerate import Accelerator, InitProcessGroupKwargs -else: - from unittest.mock import Mock - - Accelerator = InitProcessGroupKwargs = Mock() - -if is_package_available("nanotron"): - from nanotron import distributed as dist - from nanotron.parallel.context import ParallelContext - - from lighteval.models.nanotron.nanotron_model import NanotronLightevalModel - - -import logging - - -logger = logging.getLogger(__name__) - - -class ParallelismManager(Enum): - ACCELERATE = auto() - NANOTRON = auto() - TGI = auto() - OPENAI = auto() - VLLM = auto() - CUSTOM = auto() - NONE = auto() - SGLANG = auto() - - -@dataclass -class PipelineParameters: - launcher_type: ParallelismManager - # Env parameters - job_id: int = 0 - dataset_loading_processes: int = 1 - nanotron_checkpoint_path: str | None = None # only for nanotron models - # Dataset - custom_tasks_directory: str | None = None - num_fewshot_seeds: int = 1 - max_samples: int | None = None - cot_prompt: str | None = None - remove_reasoning_tags: bool = True - reasoning_tags: str | list[tuple[str, str]] = "[('', '')]" - load_responses_from_details_date_id: str | None = None - bootstrap_iters: int = 1000 - load_tasks_multilingual: bool = False - - def __post_init__(self): # noqa C901 - if not isinstance(self.reasoning_tags, list): - try: - self.reasoning_tags = ast.literal_eval(self.reasoning_tags) - except ValueError as e: - raise ValueError( - "reasoning_tags must be a list of pair tuples, e.g. [('start_tag', 'end_tag'), ...]. " - f"Got {self.reasoning_tags} instead, which caused parsing error {e}." - ) - - # Make sure format is correct - if not all(isinstance(tag, tuple) and len(tag) == 2 for tag in self.reasoning_tags): - raise ValueError( - "reasoning_tags must be a list of pair tuples, e.g. [('start_tag', 'end_tag'), ...]. " - f"Got {self.reasoning_tags} instead." - ) - - -class Pipeline: - def __init__( - self, - tasks: str, - pipeline_parameters: PipelineParameters, - evaluation_tracker: EvaluationTracker, - model_config: ModelConfig | None = None, - model=None, - metric_options=None, - ): - if not (model or model_config): - raise ValueError("Must provide either a model or model config when creating a pipeline.") - - self.pipeline_parameters = pipeline_parameters - if self.pipeline_parameters.max_samples: - logger.warning( - "--max_samples WAS SET. THESE NUMBERS ARE ONLY PARTIAL AND SHOULD NOT BE USED FOR COMPARISON UNLESS YOU KNOW WHAT YOU ARE DOING." - ) - - self.launcher_type = self.pipeline_parameters.launcher_type - self._metric_options = metric_options or {} - self.evaluation_tracker = evaluation_tracker - - # We init tasks first to fail fast if one is badly defined - self._init_random_seeds() - self._init_tasks_and_requests(tasks=tasks) - - self.model_config = model_config - self.accelerator, self.parallel_context = self._init_parallelism_manager() - self.model = self._init_model(model_config, model) - # Must occur after model and task init - self.model._cache._init_registry(self.registry) - # Must occur after model init - self._init_accelerator_seeds() - - self.evaluation_tracker.general_config_logger.log_model_info(model_config=self.model.config) - - # Final results - self.final_dict: dict | None = None - - def _init_parallelism_manager(self): - accelerator, parallel_context = None, None - if self.launcher_type == ParallelismManager.ACCELERATE: - if not is_package_available("accelerate"): - raise ValueError("You are trying to launch an accelerate model, but accelerate is not installed") - accelerator = Accelerator(kwargs_handlers=[InitProcessGroupKwargs(timeout=timedelta(seconds=3000))]) - test_all_gather(accelerator=accelerator) - elif self.launcher_type == ParallelismManager.NANOTRON: - if not is_package_available("nanotron"): - raise ValueError("You are trying to launch a nanotron model, but nanotron is not installed") - dist.initialize_torch_distributed() - parallel_context = ParallelContext( - tensor_parallel_size=self.model_config.lighteval_config.parallelism.tp, - pipeline_parallel_size=self.model_config.lighteval_config.parallelism.pp, - data_parallel_size=self.model_config.lighteval_config.parallelism.dp, - ) - test_all_gather(parallel_context=parallel_context) - - return accelerator, parallel_context - - def _init_model(self, model_config, model): - logger.info("--- LOADING MODEL ---") - - if model is not None and model_config is not None: - if isinstance(model, LightevalModel): - raise ValueError( - "You are trying to provide both a LightevalModel and a model config. Please provide only one of them." - ) - return TransformersModel.from_model( - model=model, - config=model_config, - accelerator=self.accelerator, - ) - - elif model is not None: - if isinstance(model, LightevalModel): - return model - raise ValueError("If not providing a model_config, you need to provide a Lighteval model.") - - elif model_config is not None: - if self.parallel_context: - return NanotronLightevalModel( - checkpoint_path=os.path.dirname(self.pipeline_parameters.nanotron_checkpoint_path) - if self.pipeline_parameters.nanotron_checkpoint_path - else "", - nanotron_config=model_config, - parallel_context=self.parallel_context, - debug_one_layer_model=False, - model_class=None, - ) - else: - return load_model(config=model_config) - - def _init_tasks_and_requests(self, tasks: str): - logger.info("--- LOADING TASKS ---") - - # The registry contains all the potential tasks - self.registry = Registry( - tasks=tasks, - load_multilingual=self.pipeline_parameters.load_tasks_multilingual, - custom_tasks=self.pipeline_parameters.custom_tasks_directory, - ) - - # load the tasks from the configs and their datasets - self.tasks_dict: dict[str, LightevalTask] = self.registry.load_tasks() - LightevalTask.load_datasets(self.tasks_dict, self.pipeline_parameters.dataset_loading_processes) - self.documents_dict = { - task.full_name: task.get_docs(self.pipeline_parameters.max_samples) for _, task in self.tasks_dict.items() - } - - self.sampling_docs = collections.defaultdict(list) - for _, docs in self.documents_dict.items(): - for doc in docs: - for sampling in doc.sampling_methods: - self.sampling_docs[sampling].append(doc) - - # If there are metric_options defined from the yaml file, - # review if they have to be updated. - if self._metric_options: - self._update_num_samples(list(self.tasks_dict.values())) - - self.evaluation_tracker.task_config_logger.log(self.tasks_dict) - - def _update_num_samples(self, tasks: list[LightevalTask]): - """Helper function to update the num_samples of a given metric via the yaml file. - As it has to be done at the metric level, it's better to update the value per metric. - It will add a num_samples to the already defined metrics' num_samples if defined in the yaml file. - As later when constructing the requests the max is taken over the num_samples, this is valid. - """ - for task in tasks: - for metric in task.metrics: - if metric_data := self._metric_options.get(metric.metric_name, None): - num_samples = metric_data.get("num_samples", None) - if num_samples: - task.num_samples = [num_samples] - - def _init_random_seeds(self): - logger.info("--- INIT SEEDS ---") - random.seed(1234) - np.random.seed(1234) - - def _init_accelerator_seeds(self): - if self.accelerator is not None: - self.accelerator.wait_for_everyone() - if self.parallel_context is not None: - dist.barrier() - - def is_main_process(self): - if self.accelerator: - return self.accelerator.is_main_process - if self.parallel_context: - return dist.get_rank(self.parallel_context.world_pg) == 0 - return True - - def evaluate(self): - self.evaluation_tracker.general_config_logger.log_args_info( - num_fewshot_seeds=self.pipeline_parameters.num_fewshot_seeds, - max_samples=self.pipeline_parameters.max_samples, - job_id=str(self.pipeline_parameters.job_id), - ) - - if self.pipeline_parameters.load_responses_from_details_date_id: - try: - outputs = self._load_responses_from_details() - except FileNotFoundError as e: - logger.warning( - f"No responses found for {self.pipeline_parameters.load_responses_from_details_date_id} in details directory: {e}. Running model instead." - ) - outputs = self._run_model() - else: - outputs = self._run_model() - - if self.is_main_process(): - self._post_process_outputs(outputs) - self._compute_metrics(outputs) - - self.evaluation_tracker.general_config_logger.log_end_time() - self.evaluation_tracker.metrics_logger.aggregate( - task_dict=self.tasks_dict, bootstrap_iters=self.pipeline_parameters.bootstrap_iters - ) - self.evaluation_tracker.details_logger.aggregate() - - async def _run_model_async(self): - outputs = {} - for sampling_method, docs in self.sampling_docs.items(): - logger.info(f"Running {sampling_method} requests") - match sampling_method: - case SamplingMethod.GENERATIVE: - model_outputs = await self.model.greedy_until(docs) - outputs[sampling_method] = model_outputs - case SamplingMethod.LOGPROBS: - model_outputs = await self.model.loglikelihood(docs) - outputs[sampling_method] = model_outputs - - return outputs - - def _run_model_sync(self): - # Running all requests depending on the model call type (log likelihood, generative, ...) - # to be able to batch them - outputs = {} - for sampling_method, docs in self.sampling_docs.items(): - logger.info(f"Running {sampling_method} requests") - match sampling_method: - case SamplingMethod.GENERATIVE: - model_outputs = self.model.greedy_until(docs) - outputs[sampling_method] = model_outputs - case SamplingMethod.LOGPROBS: - model_outputs = self.model.loglikelihood(docs) - outputs[sampling_method] = model_outputs - case SamplingMethod.PERPLEXITY: - model_outputs = self.model.loglikelihood_rolling(docs) - outputs[sampling_method] = model_outputs - - return outputs - - def _run_model(self): - # Running all requests depending on the model call type (log likelihood, generative, ...) - # to be able to batch them - logger.info("--- RUNNING MODEL ---") - - if self.model.is_async: - outputs = asyncio.run(self._run_model_async()) - else: - outputs = self._run_model_sync() - - # Cleaning up the model before running metrics - self.model.cleanup() - - return outputs - - def _post_process_outputs(self, sampling_method_responses: dict[str, list[ModelResponse]]): - # Removes reasoning tags if needed - logger.info("--- POST-PROCESSING MODEL RESPONSES ---") - - if self.pipeline_parameters.remove_reasoning_tags: - for _, responses in sampling_method_responses.items(): - for response in responses: - response.text_post_processed = [ - remove_reasoning_tags( - text=text, - tag_pairs=self.pipeline_parameters.reasoning_tags, - ) - for text in response.text - ] - - def _compute_metrics(self, sampling_method_responses: dict[str, list[ModelResponse]]): - # To compute the metrics we first group the samples and task and then by metrics. - # This way we can batch the metrics computation for each task and metric category - - # This variable will hold the samples grouped by task and metric category - # example: - # task_metric_category_groups = { - # "gsm8k_1": { - # "GENERATIVE": [ - # (doc1, response1), (doc2, response2), ..., - # } - # "LOGLIKELIHOOD": [ - # (doc1, response1), (doc2, response2), ..., - # ] - logger.info("--- COMPUTING METRICS ---") - task_metric_category_groups = collections.defaultdict(lambda: collections.defaultdict(list)) - - for sampling_method, model_responses in sampling_method_responses.items(): - for doc, model_reponse in zip(self.sampling_docs[sampling_method], model_responses): - task_metric_category_groups[doc.task_name][sampling_method].append((doc, model_reponse)) - - for task_name, samples_per_method in task_metric_category_groups.items(): - task: LightevalTask = self.tasks_dict[task_name] - for sampling_method, samples in samples_per_method.items(): - metric_category_metrics = [metric for metric in task.metrics if metric.category == sampling_method] - - docs = [doc for doc, _ in samples] - responses = [response for _, response in samples] - - outputs = apply_metric( - docs=docs, - responses=responses, - metrics=metric_category_metrics, - ) - - for output, doc, response in zip(outputs, docs, responses): - self.evaluation_tracker.metrics_logger.log(task_name, output) - self.evaluation_tracker.details_logger.log(task_name, doc, response, output) - - def _load_responses_from_details(self): - logger.info("--- LOADING RESPONSES FROM DETAILS ---") - model_responses = {} - tasks_names = list(self.tasks_dict.keys()) - sampling_methods = list(self.sampling_docs.keys()) - - if len(sampling_methods) > 1: - raise ValueError( - "Loading responses from details when there are multiple request types is currently not supported" - ) - - assert self.pipeline_parameters.load_responses_from_details_date_id is not None - - details_datasets = self.evaluation_tracker.load_details_datasets( - self.pipeline_parameters.load_responses_from_details_date_id, tasks_names - ) - - for _, dataset in tqdm(details_datasets.items(), desc="Loading responses from details for tasks"): - for sampling_method in sampling_methods: - model_responses[sampling_method] = [ - ModelResponse(**model_response["model_response"]) for model_response in dataset - ] - - return model_responses - - def save_and_push_results(self): - logger.info("--- SAVING AND PUSHING RESULTS ---") - if self.is_main_process(): - self.evaluation_tracker.save() - - def _init_final_dict(self): - if self.is_main_process(): - if self.final_dict is None: - self.final_dict = self.evaluation_tracker.generate_final_dict() - - def show_results(self): - logger.info("--- DISPLAYING RESULTS ---") - self._init_final_dict() - if self.is_main_process(): - print(make_results_table(self.final_dict)) - - def get_results(self): - self._init_final_dict() - return self.final_dict - - def get_details(self): - return self.evaluation_tracker.details_logger.details diff --git a/src/lighteval/py.typed b/src/lighteval/py.typed deleted file mode 100644 index e69de29bb..000000000 diff --git a/src/lighteval/tasks/__init__.py b/src/lighteval/tasks/__init__.py deleted file mode 100644 index e3e34484b..000000000 --- a/src/lighteval/tasks/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -""" -Automatically imports all task configs from the tasks/ directory. -This module dynamically loads all Python files in tasks/ and exposes their LightevalTaskConfig objects. -""" diff --git a/src/lighteval/tasks/lighteval_task.py b/src/lighteval/tasks/lighteval_task.py deleted file mode 100644 index 5e9bac215..000000000 --- a/src/lighteval/tasks/lighteval_task.py +++ /dev/null @@ -1,468 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import functools -import logging -import random -from dataclasses import asdict, dataclass, field -from typing import Callable, Mapping, Sequence - -from datasets import DatasetDict, load_dataset -from huggingface_hub import TextGenerationInputGrammarType -from inspect_ai.dataset import Sample -from multiprocess import Pool -from pytablewriter import MarkdownTableWriter - -from lighteval.metrics.metrics import Metrics -from lighteval.metrics.metrics_sample import SamplingMetric -from lighteval.metrics.utils.metric_utils import Metric -from lighteval.tasks.prompt_manager import FewShotSampler -from lighteval.tasks.requests import ( - Doc, -) -from lighteval.utils.utils import ListLike, as_list - - -logger = logging.getLogger(__name__) - - -@dataclass -class LightevalTaskConfig: - """Configuration dataclass for a LightevalTask. - - This class stores all the configuration parameters needed to define and run - an evaluation task, including dataset information, prompt formatting, - evaluation metrics, and generation parameters. - - Args: - name (str): Short name of the evaluation task. - prompt_function (Callable[[dict, str], Doc]): Function that converts dataset - row to Doc objects for evaluation. Takes a dataset row dict and task - name as input. - hf_repo (str): HuggingFace Hub repository path containing the evaluation dataset. - hf_data_files (str | Sequence[str] | Mapping[str, str | Sequence[str]] | None): - Data files to load. Same as `data_files` argument of `datasets.load_dataset`. - hf_subset (str): Dataset subset/configuration name to use for this task. - metrics (ListLike[Metric | Metrics]): List of metrics or metric enums to compute for this task. - - Dataset Configuration: - hf_revision (str | None, optional): Specific dataset revision to use. - Defaults to None (latest). - hf_filter (Callable[[dict], bool] | None, optional): Filter function to - apply to dataset items. Defaults to None. - hf_avail_splits (ListLike[str], optional): Available dataset splits. - Defaults to ["train", "validation", "test"]. - - Evaluation Splits: - evaluation_splits (ListLike[str], optional): Dataset splits to use for - evaluation. Defaults to ["validation"]. - few_shots_split (str | None, optional): Split to sample few-shot examples - from. Defaults to None. - few_shots_select (str | None, optional): Method for selecting few-shot - examples. Defaults to None. - - Generation Parameters: - generation_size (int | None, optional): Maximum token length for generated - text. Defaults to None. - generation_grammar (TextGenerationInputGrammarType | None, optional): Grammar - for structured text generation. Only available for TGI and Inference - Endpoint models. Defaults to None. - stop_sequence (ListLike[str] | None, optional): Sequences that stop text - generation. Defaults to None. - num_samples (list[int] | None, optional): Number of samples to generate - per input. Defaults to None. - - Task Configuration: - version (int, optional): Task version number. Increment when dataset or - prompt changes. Defaults to 0. - num_fewshots (int, optional): Number of few-shot examples to include. - Defaults to 0. - truncate_fewshots (bool, optional): Whether to truncate few-shot examples. - Defaults to False. - must_remove_duplicate_docs (bool, optional): Whether to remove duplicate - documents. Defaults to False. - - Document Tracking: - original_num_docs (int, optional): Total number of documents in the task. - Defaults to -1. - effective_num_docs (int, optional): Number of documents actually used - in evaluation. Defaults to -1. - """ - - name: str - prompt_function: Callable[ - [dict, str], Doc - ] # The prompt function should be used to map a line in the dataset to a Sample - hf_repo: str - hf_subset: str - metrics: ListLike[Metric | Metrics] # Accept both Metric objects and Metrics enums - hf_data_files: str | Sequence[str] | Mapping[str, str | Sequence[str]] | None = None - - # Inspect AI compatible parameters - solver: None = None - scorer: None = None - sample_fields: Callable[[dict], Sample] | None = None - sample_to_fewshot: Callable[[Sample], str] | None = None - filter: Callable[[dict], bool] | None = None - - # Additional hf dataset config - hf_revision: str | None = None - hf_filter: Callable[[dict], bool] | None = None - hf_avail_splits: ListLike[str] = field(default_factory=lambda: ["train", "validation", "test"]) - - # Splits - evaluation_splits: ListLike[str] = field(default_factory=lambda: ["validation"]) - few_shots_split: str | None = None - few_shots_select: str | None = None - - # Generation args - generation_size: int | None = None - generation_grammar: TextGenerationInputGrammarType | None = None - stop_sequence: ListLike[str] | None = None - num_samples: list[int] | None = None - - original_num_docs: int = -1 - effective_num_docs: int = -1 - - must_remove_duplicate_docs: bool = False - - num_fewshots: int = 0 - - version: int = 0 - - def __post_init__(self): - # If we got a Metrics enums instead of a Metric, we convert - self.metrics = [metric.value if isinstance(metric, Metrics) else metric for metric in self.metrics] - # Convert list to tuple for hashing - self.metrics = tuple(self.metrics) - self.hf_avail_splits = tuple(self.hf_avail_splits) - self.evaluation_splits = tuple(self.evaluation_splits) - self.stop_sequence = self.stop_sequence if self.stop_sequence is not None else () - self.full_name = f"{self.name}|{self.num_fewshots}" # todo clefourrier: this is likely incorrect - - def __str__(self, lite: bool = False): # noqa: C901 - md_writer = MarkdownTableWriter() - md_writer.headers = ["Key", "Value"] - - # These keys change through time - to_ignore = ["original_num_docs", "effective_num_docs"] - - values = [] - - for k, v in asdict(self).items(): - if lite and k in to_ignore: - continue - if k == "metrics": - for ix, metrics in enumerate(v): - for metric_k, metric_v in metrics.items(): - if isinstance(metric_v, functools.partial): - func_name = getattr(metric_v.func, "__name__", str(metric_v.func)) - repr_v = f"partial({func_name}, ...)" - elif isinstance(metric_v, Callable): - repr_v = getattr(metric_v, "__name__", repr(metric_v)) - elif isinstance(metric_v, Metric.get_allowed_types_for_metrics()): - repr_v = str(metric_v) - else: - repr_v = repr(metric_v) - values.append([f"{k} {ix}: {metric_k}", repr_v]) - - else: - if isinstance(v, functools.partial): - func_name = getattr(v.func, "__name__", str(v.func)) - values.append([k, f"partial({func_name}, ...)"]) - elif isinstance(v, Callable): - values.append([k, getattr(v, "__name__", repr(v))]) - else: - values.append([k, repr(v)]) - - md_writer.value_matrix = values - - return md_writer.dumps() - - def print(self, lite: bool = False): - print(str(self, lite)) - - -class LightevalTask: - def __init__( - self, - config: LightevalTaskConfig, - ): - """Initialize a LightEval task. - - Args: - config (dict): configuration dictionary containing - task-specific settings (from the task_table.json file). - """ - self.config = config - self.name = config.name - self.version = config.version - self.dataset_config = config - - self.full_name = config.full_name - - # Dataset info - self.dataset_path = config.hf_repo - self.data_files = config.hf_data_files - self.dataset_config_name = config.hf_subset - self.dataset_revision = config.hf_revision - self.dataset_filter = config.hf_filter - self.dataset: DatasetDict | None = None # Delayed download - self.evaluation_split = as_list(config.evaluation_splits) - self._docs = None - - self._fewshot_docs = None - self.fewshot_split: str | None = config.few_shots_split or self.get_first_possible_fewshot_splits( - config.hf_avail_splits or [] - ) - self.fewshot_selection = config.few_shots_select - self.must_remove_duplicate_docs = config.must_remove_duplicate_docs - - self.formatter = config.prompt_function - self.fewshot_sampler = FewShotSampler(self) - - # Metrics - self.metrics = config.metrics - self.sampling_methods = list({metric.category for metric in self.metrics}) - - # generation parameters - self.generation_size = config.generation_size - self.generation_grammar = config.generation_grammar - self.stop_sequence = config.stop_sequence - - # We assume num_samples always contains 1 (for base generative evals) - self.num_samples = [1] - for metric in self.metrics: - if isinstance(metric.sample_level_fn, SamplingMetric): - # Update the number of samples to generate using the information in the metric name - self.num_samples.append(metric.sample_level_fn.num_samples()) - - def get_first_possible_fewshot_splits(self, available_splits: ListLike[str]) -> str | None: - """Parses the possible fewshot split keys in order: train, then validation - keys and matches them with the available keys. Returns the first - available. - - Returns: - str: the first available fewshot splits or None if nothing is available - """ - # Possible few shot splits are the available splits not used for evaluation - possible_fewshot_splits = [k for k in available_splits if k not in self.evaluation_split] - stored_splits = [] - - # We look at these keys in order (first the training sets, then the validation sets) - allowed_splits = ["train", "dev", "valid", "default"] - for allowed_split in allowed_splits: - # We do a partial match of the allowed splits - available_splits = [k for k in possible_fewshot_splits if allowed_split in k] - stored_splits.extend(available_splits) - - if len(stored_splits) > 0: - return stored_splits[0] - - logger.warning(f"Careful, the task {self.name} is using evaluation data to build the few shot examples.") - return None - - def _get_docs_from_split(self, splits: list[str], few_shots=False) -> list[Doc]: - """Get the documents from the dataset for the given keys (splits). - - Args: - splits (list[str]): List of dataset splits to process (e.g. ["train", "dev"]) - few_shots (bool, optional): Whether the documents are used for few-shot - examples. This affects how the formatter processes the items. Defaults to False. - - Returns: - list[Doc]: List of documents. - """ - if self.dataset is None: - self.dataset = self.download_dataset_worker(self) - - assert self.dataset is not None, f"Dataset {self.dataset_path} not found." - - docs = [] - for split in splits: - for ix, item in enumerate(self.dataset[split]): - # Some tasks formatting is applied differently when the document is used for fewshot examples - # vs when it's used for the actual prompt. That's why we store whether we are currently using the - # doc for a fewshot sample (few_shots=True) or not, which then leads to the creation of a different Doc. - item["__few_shots"] = few_shots - # Some tasks require to know which is the current item index in order to apply a different prompt template - item["__index"] = ix - doc = self.formatter(item, self.name) - # Skip if formatter returns None (e.g., to filter out certain samples) - if doc is None or doc == []: - continue - - doc.id = str(ix) - - # Transfer task-level generation parameters to the document - doc.generation_grammar = self.generation_grammar - doc.generation_size = self.generation_size - doc.stop_sequences = self.stop_sequence - - docs.append(doc) - - return docs - - def remove_duplicate_docs(self, docs: list[Doc]) -> list[Doc]: - seen_examples, res = set(), [] - for doc in docs: - if str(doc) not in seen_examples: - res.append(doc) - seen_examples.add(str(doc)) - return res - - def fewshot_docs(self) -> list[Doc]: - """Returns the few shot documents. If the few shot documents are not - available, it gets them from the few shot split or the evaluation split. - - Returns: - list[Doc]: Documents that will be used for few shot examples. One - document = one few shot example. - """ - if self._fewshot_docs is None: - self._fewshot_docs = [] - - # If we have no available few shot split, the few shot data is the eval data! - if self.fewshot_split is None: - self._fewshot_docs = self._get_docs_from_split(self.evaluation_split, few_shots=True) - else: # Normal case - self._fewshot_docs = self._get_docs_from_split([self.fewshot_split], few_shots=True) - - return self._fewshot_docs - - def eval_docs(self) -> list[Doc]: - """Returns the evaluation documents. - - Returns: - list[Doc]: Evaluation documents. - """ - if self._docs is None: - self._docs = self._get_docs_from_split(self.evaluation_split) - if self.must_remove_duplicate_docs: - self._docs = self.remove_duplicate_docs(self._docs) - return self._docs - - def get_docs(self, max_samples: int | None = None) -> list[Doc]: - """Get evaluation documents with few-shot examples and generation parameters configured. - - Retrieves evaluation documents, optionally limits the number of samples, - shuffles them for reproducibility, and configures each document with - few-shot examples and generation parameters for evaluation. - - Args: - max_samples (int | None, optional): Maximum number of documents to return. - If None, returns all available documents. Defaults to None. - - Returns: - list[Doc]: List of documents ready for evaluation with few-shot examples - and generation parameters configured. - - Raises: - ValueError: If no documents are available for evaluation. - """ - eval_docs = self.eval_docs() - - if len(eval_docs) == 0: - raise ValueError(f"Task {self.name} has no documents to evaluate skipping.") - - n_samples = min(max_samples, len(eval_docs)) if max_samples else len(eval_docs) - rnd = random.Random() - rnd.seed(42) - rnd.shuffle(eval_docs) - - docs = [] - - for doc in eval_docs[:n_samples]: - num_fewshots = self.dataset_config.num_fewshots - doc.task_name = self.full_name - doc.fewshot_samples = self.fewshot_sampler.sample_fewshot_examples( - num_fewshots, 0, formatted_doc=doc, sampler=rnd - ) - doc.sampling_methods.extend(self.sampling_methods) - doc.generation_size = self.generation_size - doc.use_logits = doc.use_logits if doc.use_logits is not None else True - doc.stop_sequences = self.stop_sequence - doc.num_samples = max(self.num_samples) - docs.append(doc) - - return docs - - def aggregation(self): - """Return a dict with metric name and its aggregation function for all - metrics - """ - aggregations = {} - for metric in self.metrics: - aggregations.update(metric.get_corpus_aggregations()) - return aggregations - - @staticmethod - def load_datasets(tasks: dict[str, "LightevalTask"], dataset_loading_processes: int = 1) -> None: - """Load datasets from the HuggingFace Hub for the given tasks. - - Args: - tasks (dict[str, LightevalTask]): Dictionary mapping task names to task objects. - dataset_loading_processes (int, optional): Number of processes to use for - parallel dataset loading. Defaults to 1 (sequential loading). - """ - if dataset_loading_processes <= 1: - # Useful for the test suite: we can mock loading tasks by overwriting the - # individual download_dataset_worker functions - datasets = [task.download_dataset_worker(task) for task in tasks.values()] - else: - with Pool(processes=dataset_loading_processes) as pool: - datasets = pool.starmap( - LightevalTask.download_dataset_worker, - [tasks.values()], - ) - - for task, dataset in zip(tasks, datasets): - tasks[task].dataset = dataset - - @staticmethod - def download_dataset_worker( - task: "LightevalTask", - ) -> DatasetDict: - """Worker function to download a dataset from the HuggingFace Hub. - - Downloads the dataset specified in the task configuration, optionally - applies a filter if configured, and returns the dataset dictionary. - This method is designed to be used for parallel dataset loading. - - Args: - task (LightevalTask): The task object containing dataset configuration. - - Returns: - DatasetDict: The loaded dataset dictionary containing all splits. - """ - dataset = load_dataset( - path=task.dataset_path, - name=task.dataset_config_name, - revision=task.dataset_revision, - data_files=task.data_files, - ) - - if task.dataset_filter is not None: - dataset = dataset.filter(task.dataset_filter) - - # It returns DatasetDict because we don't specify a split - return dataset # type: ignore diff --git a/src/lighteval/tasks/multilingual/__init__.py b/src/lighteval/tasks/multilingual/__init__.py deleted file mode 100644 index a732db8d0..000000000 --- a/src/lighteval/tasks/multilingual/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. diff --git a/src/lighteval/tasks/multilingual/adapters.py b/src/lighteval/tasks/multilingual/adapters.py deleted file mode 100644 index d4e2dfc41..000000000 --- a/src/lighteval/tasks/multilingual/adapters.py +++ /dev/null @@ -1,300 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import os -import re -from string import ascii_uppercase - -import numpy as np -from langcodes import standardize_tag - -from lighteval.tasks.multilingual.utils.adapters_utils import ( - extract_answers_from_string, - multichoice_join, - multichoice_to_single_choice, -) -from lighteval.tasks.templates.continuation import ContinuationInput -from lighteval.tasks.templates.multichoice import MCQInput -from lighteval.tasks.templates.qa import QAInput -from lighteval.tasks.templates.utils.formatting_utils import PUNCT -from lighteval.tasks.templates.utils.formulation import CFFormulation, Formulation -from lighteval.tasks.templates.utils.translation_literals import TranslationLiterals -from lighteval.utils.language import Language - - -M3_EXAM_ANSWER_PREFIX_RE = re.compile(r"^\([A-Da-d1-5๑๒๓๔๕]\)\s*|^[A-Da-e1-5๑๒๓๔๕][.．।。]\s*") -WHITESPACES = " \t\n\r\f\v" - - -def get_m3exam_adapter(lang: Language, line: dict) -> MCQInput | None: - letter_indices = "๑๒๓๔๕" if lang == "th" else ascii_uppercase - is_number_based = line["answer_text"].isdigit() - clean_options = [M3_EXAM_ANSWER_PREFIX_RE.sub("", c) for c in line["options"]] - gold_idx = int(line["answer_text"]) - 1 if is_number_based else letter_indices.index(line["answer_text"].upper()) - - if not all(len(c) > 0 for c in clean_options) or gold_idx >= len(clean_options): - return None - - return { - "choices": clean_options, - "gold_idx": gold_idx, - "question": line["question_text"], - "context": line["background"], - } - - -def thai_exams_adapter(line: dict) -> MCQInput | None: - pos_letters = [letter.lower() for letter in ascii_uppercase[:5]] - - letter_to_choices = {letter: line[letter] for letter in pos_letters if letter in line} - if any(opt.strip() == "" for opt in letter_to_choices.values()): - return None - - gold_index = list(letter_to_choices.keys()).index(line["answer"]) - return { - "question": line["question"], - "choices": list(letter_to_choices.values()), - "gold_idx": gold_index, - } - - -def alghafa_adapter(line: dict) -> MCQInput | None: - answer_index = int(line["label"]) - choices_keys = [key for key in line.keys() if key not in ["query", "label", "__few_shots"]] - choices = [line[key] for key in choices_keys] - return { - "question": line["query"], - "choices": choices, - "gold_idx": answer_index, - } - - -def sciqa_adapter(line: dict) -> MCQInput | None: - # Randomly generate the correct answer index - gold_idx = np.random.randint(0, 4) - incorrect_answers = [line["distractor1"], line["distractor2"], line["distractor3"]] - choices = incorrect_answers[:gold_idx] + [line["correct_answer"]] + incorrect_answers[gold_idx:] - return { - "question": line["question"], - "choices": choices, - "gold_idx": gold_idx, - "context": line["support"], - } - - -def ceval_adapter(lang: Language, formulation: Formulation, line: dict) -> MCQInput | None: - # All ceval tasks ends with ____(。?) - # Some can follow with with possible answers in format - # (①|②|③|④)text - # We must thus remove ___ and extract the answers from the second part of input text - - translation_literals = TranslationLiterals(lang) - choices = [line["A"], line["B"], line["C"], line["D"]] - - # We found the new line variant to be the best for CF formulation, however for MCF this doesn't work well - # because possible options presentation - join_variant = "NEW_LINE" if isinstance(formulation, CFFormulation) else "COMMA" - - parts = line["question"].rsplit("____", maxsplit=1) - cleaned_question = parts[0].rstrip(WHITESPACES) - possible_answers_part = parts[1].lstrip(PUNCT + WHITESPACES).rstrip() - gold_index = ascii_uppercase.index(line["answer"]) - - # We only attempt to extract answers if the answers are a chinese numbers - answer_prefixes = [answer.replace("和", "").strip() for answer in choices] - answer_prefixes_set = set("".join(answer_prefixes)) - - maybe_extracted_answers = ( - extract_answers_from_string(possible_answers_part, list(answer_prefixes_set)) - if answer_prefixes_set.issubset("①②③④⑤⑥") - else None - ) - if maybe_extracted_answers: - start_index, prefix_answer_map = maybe_extracted_answers - # Here we don't expect anything to be in front of the answers from second part - assert start_index == 0, f"Start index is not 0: {start_index}" - - choices_groups = [[prefix_answer_map.get(prefix) for prefix in prefixes] for prefixes in answer_prefixes] - # If we failed to extract some of the answers we discard the sample - if any(choice is None for choices in choices_groups for choice in choices): - return None - - choices = [multichoice_join(mc, join_variant, translation_literals) for mc in choices_groups] - else: - # If the second part is not list of answers we put it back - cleaned_question = f"{cleaned_question} {possible_answers_part}" if possible_answers_part else cleaned_question - - # Lastly make it into question: - cleaned_question = f"{cleaned_question.strip().rstrip(PUNCT)}{translation_literals.question_mark}" - - # If we still have only the numbers in the answers or we have just single choice we discard this sample - if set("".join(choices).replace("和", "").strip()).issubset("①②③④⑤⑥") or len(choices) <= 1: - return None - - return { - "question": cleaned_question, - "choices": choices, - "gold_idx": gold_index, - } - - -def agieval_adapter(lang: Language, formulation: Formulation, line: dict) -> MCQInput | None: - translation_literals = TranslationLiterals(lang) - - # We found the new line variant to be the best for CF formulation, however for MCF this doesn't work well - # because of possible options presentation - join_variant = "NEW_LINE" if isinstance(formulation, CFFormulation) else "COMMA" - - # Remove the question at the start as it's added by template - context, rest = line["query"].split("问题：", maxsplit=1) - - # Remove the options as we build them ourselves - question, _ = rest.split(" 选项：", maxsplit=1) - question = question.lstrip() - original_choices = line["choices"] - cleaned_choices = [M3_EXAM_ANSWER_PREFIX_RE.sub("", c).strip() for c in original_choices] - gold_index = line["gold"] - - # Here is the most tricky part. In some subsets (e.g. gaokai-history) the answers can be the chinese digits only. - # This would break the CF formulation and we thus try to extract the full answers from the question. - # Example - # Quesiton: 问题：在中美关系的发展中，台湾问题是一大障碍，在扫除这一障碍的过程中，取得突破性进展的事件包括（　　）①中国恢复联合国席位 ②尼克松总统访华③中美两国正式建交 ④邓小平访问美国。 选项：(A)①② (B)①③ (C)②③ (D)③④ 答案：从A到D, 我们应选择 - # Answers: [ "(A)①②", "(B)①③", "(C)②③", "(D)③④" ] - - answer_prefixes = [answer.replace("和", "").strip() for answer in cleaned_choices] - answer_prefixes_set = set("".join(answer_prefixes)) - - # We only attempt to extract answers if the answers are chinese numbers - # We don't want to rstrip original question as we might have failed the extraction - maybe_extracted_answers = ( - extract_answers_from_string(question.rstrip(PUNCT + WHITESPACES), list(answer_prefixes_set)) - if answer_prefixes_set.issubset("①②③④⑤⑥") - else None - ) - if maybe_extracted_answers: - start_index, prefix_answer_map = maybe_extracted_answers - question = question[:start_index] - choices_groups = [[prefix_answer_map.get(prefix) for prefix in prefixes] for prefixes in answer_prefixes] - if any(choice is None for choices in choices_groups for choice in choices): - return None - cleaned_choices = [multichoice_join(mc, join_variant, translation_literals) for mc in choices_groups] - - # Agi-eval is multi-choice but we convert it to single choice - cleaned_choices, gold_index = multichoice_to_single_choice( - cleaned_choices, gold_index, join_variant, translation_literals - ) - question = question.strip() - - # If the answers still only contian the chinese numbers or we have just single choice we discard this sample - if ( - set("".join(cleaned_choices).replace("和", "").strip()).issubset("①②③④⑤⑥") - or len(cleaned_choices) <= 1 - or any(len(choice.strip()) == 0 for choice in cleaned_choices) - ): - return None - - return { - "question": question, - "choices": cleaned_choices, - "gold_idx": gold_index, - "context": context, - } - - -def xcodah_adapter(lang: Language, line: dict) -> MCQInput | None: - translation_literals = TranslationLiterals(lang) - - gold_index = line["question"]["choices"]["label"].index(line["answerKey"]) - # All the choices have already common prefix "baken in" so we have to remove to get clearer signal - # Extract common prefix from choices - choices = line["question"]["choices"]["text"] - common_prefix = os.path.commonprefix(choices) - - # Backtract to first space to get good tokenization - first_word = common_prefix.rfind(translation_literals.word_space) - - # If there is no word_space we shouldn't remove the common prefix - common_prefix = common_prefix[:first_word] if first_word != -1 else "" - - # Remove common prefix from each choice - cleaned_choices = [choice[len(common_prefix) :] for choice in choices] - - if any(len(c.strip()) == 0 for c in cleaned_choices): - return None - - return { - "question": common_prefix, - "choices": cleaned_choices, - "gold_idx": gold_index, - } - - -def winogrand_adapter(lang: Language, line: dict) -> ContinuationInput | None: - translation_literals = TranslationLiterals(lang) - if line["sentence"].count("_") != 1: - return None - - query, end_of_target = line["sentence"].split("_") - if len(query.strip()) == 0: - return None - - options = [line["option1"], line["option2"]] - return { - "context": query, - "continuations": [f"{o}{translation_literals.word_space}{end_of_target}" for o in options], - "gold_idx": int(line["answer"]) - 1, - } - - -def get_mkqa_adapter(lang: Language, line: dict) -> QAInput | None: - lang_key = "zh_cn" if lang == Language.CHINESE else standardize_tag(lang.value) - text = line["answers"][lang_key][0]["text"] - if text is None: - return None - - aliases = line["answers"][lang_key][0]["aliases"] - answers = list(filter(lambda x: len(x.strip()) > 0, [text] + aliases)) - # Some samples are broken so this is heuristic - # e. g 'text': '七月 20, 1969', - # 'aliases': ['1', 'u', ',', '2', ' ', '6', 'l', 'y', '9', '0', 'j']}], - if len(answers) == 0 or len(answers) > 5: - return None - - return { - "question": line["queries"][lang_key], - "choices": answers, - } - - -def enem_adapter(lang: Language, line: dict) -> MCQInput | None: - if line["label"] == "Anulado": - return None - - question = line["question"] - for desc in line["description"]: - question = question.replace("[[placeholder]]", desc, 1) # Replace only first occurrence each time - - return { - "question": question, - "choices": line["alternatives"], - "gold_idx": ascii_uppercase.index(line["label"]), - } diff --git a/src/lighteval/tasks/multilingual/tasks/acva.py b/src/lighteval/tasks/multilingual/tasks/acva.py deleted file mode 100644 index c0bf6c1af..000000000 --- a/src/lighteval/tasks/multilingual/tasks/acva.py +++ /dev/null @@ -1,114 +0,0 @@ -""" -name: -Acva - -dataset: -OALL/ACVA - -abstract: -Acva multilingual benchmark. - -languages: -arabic - -tags: -knowledge, multilingual, multiple-choice - -paper: -""" - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, - MultilingualQuasiExactMatchMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.boolq import get_boolq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, -) -from lighteval.utils.language import Language - - -ACVA_SUBSET = [ - "Algeria", - "Ancient_Egypt", - "Arab_Empire", - "Arabic_Architecture", - "Arabic_Art", - "Arabic_Astronomy", - "Arabic_Calligraphy", - "Arabic_Ceremony", - "Arabic_Clothing", - "Arabic_Culture", - "Arabic_Food", - "Arabic_Funeral", - "Arabic_Geography", - "Arabic_History", - "Arabic_Language_Origin", - "Arabic_Literature", - "Arabic_Math", - "Arabic_Medicine", - "Arabic_Music", - "Arabic_Ornament", - "Arabic_Philosophy", - "Arabic_Physics_and_Chemistry", - "Arabic_Wedding", - "Bahrain", - "Comoros", - "Egypt_modern", - "InfluenceFromAncientEgypt", - "InfluenceFromByzantium", - "InfluenceFromChina", - "InfluenceFromGreece", - "InfluenceFromIslam", - "InfluenceFromPersia", - "InfluenceFromRome", - "Iraq", - "Islam_Education", - "Islam_branches_and_schools", - "Islamic_law_system", - "Jordan", - "Kuwait", - "Lebanon", - "Libya", - "Mauritania", - "Mesopotamia_civilization", - "Morocco", - "Oman", - "Palestine", - "Qatar", - "Saudi_Arabia", - "Somalia", - "Sudan", - "Syria", - "Tunisia", - "United_Arab_Emirates", - "Yemen", - "communication", - "computer_and_phone", - "daily_life", - "entertainment", -] - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"acva_{Language.ARABIC.value}:{subset}", - prompt_function=get_boolq_prompt_function( - Language.ARABIC, - lambda line: { - "question": line["question"], - "answer": line["answer"] == "صح", - }, - formulation=CFFormulation(), - ), - hf_repo="OALL/ACVA", - hf_subset=subset, - evaluation_splits=("test",), - few_shots_split="validation", - metrics=[MultilingualQuasiExactMatchMetric(Language.ARABIC, "full"), LogLikelihoodAccMetric()], - generation_size=5, - stop_sequence=("\n",), - ) - for subset in ACVA_SUBSET -] diff --git a/src/lighteval/tasks/multilingual/tasks/afri_mgsm.py b/src/lighteval/tasks/multilingual/tasks/afri_mgsm.py deleted file mode 100644 index 01b8b7334..000000000 --- a/src/lighteval/tasks/multilingual/tasks/afri_mgsm.py +++ /dev/null @@ -1,71 +0,0 @@ -""" -name: -Afri Mgsm - -dataset: -masakhane/afrimgsm - -abstract: -African MGSM: MGSM for African Languages - -languages: -amharic, ewe, french, hausa, igbo, kinyarwanda, lingala, luganda, oromo, shona, -sotho, swahili, twi, wolof, xhosa, yoruba, zulu - -tags: -math, multilingual, reasoning - -paper: -https://arxiv.org/abs/2406.03368. -""" - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"afri_mgsm_{language.value}", - prompt_function=get_qa_prompt_function( - language, - lambda line: { - "question": line["question"], - # The cot is available but we have no use: - # line["answer"] - "choices": [str(line["answer_number"])], - }, - ), - hf_repo="masakhane/afrimgsm", - hf_subset=language.value, - evaluation_splits=("test",), - few_shots_split="train", - generation_size=25, - metrics=[ - MultilingualQuasiExactMatchMetric(language, "full"), - ], - stop_sequence=("\n",), - ) - for language in [ - Language.AMHARIC, - # Language.EWE, - Language.FRENCH, - # Language.HAUSA, - # Language.IGBO, - # Language.KINYARWANDA, - # Language.LINGALA, - # Language.LUGANDA, - # Language.OROMO, - # Language.SHONA, - # Language.SOTHO, - Language.SWAHILI, - # Language.TWI, - # Language.WOLOF, - # Language.XHOSA, - Language.YORUBA, - # Language.ZULU, - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/afri_mmlu.py b/src/lighteval/tasks/multilingual/tasks/afri_mmlu.py deleted file mode 100644 index 67124f10a..000000000 --- a/src/lighteval/tasks/multilingual/tasks/afri_mmlu.py +++ /dev/null @@ -1,103 +0,0 @@ -""" -name: -Afri Mmlu - -dataset: -masakhane/afrimmlu - -abstract: -African MMLU: African Massive Multitask Language Understanding - -languages: -amharic, ewe, french, hausa, igbo, kinyarwanda, lingala, luganda, oromo, shona, -sotho, swahili, twi, wolof, xhosa, yoruba, zulu - -tags: -knowledge, multilingual, multiple-choice - -paper: -https://arxiv.org/abs/2406.03368. -""" - -from functools import partial -from string import ascii_uppercase - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbPMINorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -AFRI_MMLU_SUBSETS = [ - "elementary_mathematics", - "high_school_mathematics", - "high_school_geography", - "high_school_microeconomics", - "international_law", - "global_facts", -] - - -afri_mmlu_tasks = [ - LightevalTaskConfig( - name=f"afri_mmlu_{language.value}_{formulation.name.lower()}:{subset}", - prompt_function=get_mcq_prompt_function( - language, - lambda line: { - "question": line["question"], - "choices": line["choices"], - "gold_idx": ascii_uppercase.index(line["answer"]), - }, - formulation=formulation, - ), - hf_repo="masakhane/afrimmlu", - # Temporary until the pr is merged - hf_revision="refs/pr/1", - hf_subset=language.value, - hf_filter=partial(lambda subset, line: line["subject"] == subset, subset), - evaluation_splits=("test",), - few_shots_split="dev", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbPMINorm()), - ], - ), - ) - for subset in AFRI_MMLU_SUBSETS - for language in [ - Language.AMHARIC, - # Language.EWE, - Language.FRENCH, - # Language.HAUSA, - # Language.IGBO, - # Language.KINYARWANDA, - # Language.LINGALA, - # Language.LUGANDA, - # Language.OROMO, - # Language.SHONA, - # Language.SOTHO, - Language.SWAHILI, - # Language.TWI, - # Language.WOLOF, - # Language.XHOSA, - Language.YORUBA, - # Language.ZULU, - ] - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/afri_xnli.py b/src/lighteval/tasks/multilingual/tasks/afri_xnli.py deleted file mode 100644 index ddcad039d..000000000 --- a/src/lighteval/tasks/multilingual/tasks/afri_xnli.py +++ /dev/null @@ -1,85 +0,0 @@ -""" -name: -Afri Xnli - -dataset: -masakhane/afrixnli - -abstract: -African XNLI: African XNLI - -languages: -amharic, ewe, french, hausa, igbo, kinyarwanda, lingala, luganda, oromo, shona, -sotho, swahili, twi, wolof, xhosa, yoruba, zulu - -tags: -classification, multilingual, nli - -paper: -https://arxiv.org/abs/2406.03368. -""" - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.nli import get_nli_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"afri_xnli_{language.value}_{formulation.name.lower()}", - prompt_function=get_nli_prompt_function( - language=language, - adapter=lambda line: { - "premise": line["premise"], - "hypothesis": line["hypothesis"], - # Since we ignore the neutral label - "gold_idx": {0: 0, 2: 1}[line["label"]], - }, - relations=["entailment", "contradiction"], - formulation=formulation, - ), - hf_repo="masakhane/afrixnli", - hf_subset=language.value, - hf_filter=lambda x: int(x["label"]) in [0, 2], - evaluation_splits=("test",), - few_shots_split="validation", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=None), - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for language in [ - Language.AMHARIC, - # Language.EWE, - Language.FRENCH, - # Language.HAUSA, - # Language.IGBO, - # Language.KINYARWANDA, - # Language.LINGALA, - # Language.LUGANDA, - # Language.OROMO, - # Language.SHONA, - # Language.SOTHO, - Language.SWAHILI, - # Language.TWI, - # Language.WOLOF, - # Language.XHOSA, - Language.YORUBA, - # Language.ZULU, - ] - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] -] diff --git a/src/lighteval/tasks/multilingual/tasks/arabic.py b/src/lighteval/tasks/multilingual/tasks/arabic.py deleted file mode 100644 index 2d78f9887..000000000 --- a/src/lighteval/tasks/multilingual/tasks/arabic.py +++ /dev/null @@ -1,1021 +0,0 @@ -""" -name: -Arabic Evals - -dataset: -MBZUAI/ArabicMMLU, MBZUAI/human_translated_arabic_mmlu, OALL/Arabic_MMLU, OALL/ACVA, asas-ai/AraTrust-categorized - -abstract: -Collection of benchmarks for Arabic language. - -languages: -arabic - -tags: -knowledge, multilingual, multiple-choice - -paper: -""" - -import random -import re -from string import ascii_uppercase -from typing import Any, Dict, List, Optional, Union - -from lighteval.metrics.metrics import Metrics -from lighteval.metrics.normalizations import LogProbCharNorm -from lighteval.metrics.utils.llm_as_judge import JudgeLM -from lighteval.metrics.utils.metric_utils import Metric -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc, SamplingMethod - - -# fmt: off -LETTER_INDICES_AR = ["أ", "ب", "ج", "د", "هـ", "و", "ز", "ح", "ط", "ي", "ك", "ل", "م", "ن", "س", "ع", "ف", "ص", "ق", "ر", "ش", "ت", "ث", "خ", "ذ", "ض", "ظ", "غ"] -# fmt: on - -# ArabicMMLU -# fmt: off -ARABIC_MMLU_SUBSETS = [ - "All", "Islamic Studies", "Islamic Studies (Middle School)", "Islamic Studies (Primary School)", "Islamic Studies (High School)", "Driving Test", - "Natural Science (Middle School)", "Natural Science (Primary School)", "History (Middle School)", "History (Primary School)", "History (High School)", "General Knowledge", - "General Knowledge (Middle School)", "General Knowledge (Primary School)", "Law (Professional)", "Physics (High School)", "Social Science (Middle School)", - "Social Science (Primary School)", "Management (University)", "Arabic Language (Middle School)", "Arabic Language (Primary School)", "Arabic Language (High School)", "Political Science (University)", - "Philosophy (High School)", "Accounting (University)", "Computer Science (Middle School)", "Computer Science (Primary School)", "Computer Science (High School)", "Computer Science (University)", - "Geography (Middle School)", "Geography (Primary School)", "Geography (High School)", "Math (Primary School)", "Biology (High School)", "Economics (Middle School)", - "Economics (High School)", "Economics (University)", "Arabic Language (General)", "Arabic Language (Grammar)", "Civics (Middle School)", "Civics (High School)" -] -# fmt: on - - -def arabic_mmlu_pfn(line, task_name: str = None): - instruction = "السؤال التالي هو سؤال متعدد الإختيارات. اختر الإجابة الصحيحة:\n\n" - - # Define the mapping from Latin to Arabic letters - latin_to_arabic = {"A": "أ", "B": "ب", "C": "ج", "D": "د", "E": "هـ"} - - # Create a list of valid choices with corresponding Arabic keys - choices = [] - valid_keys_latin = [] - valid_keys_arabic = [] - - # Enumerate through the options and append the valid ones - for idx, key in enumerate(["A", "B", "C", "D", "E"]): - option = line.get(f"Option {idx + 1}") - if option: # Check if option is not null - choices.append(option) - valid_keys_latin.append(key) # Append the Latin key (A, B, C, D, E) - valid_keys_arabic.append(latin_to_arabic[key]) # Append the corresponding Arabic letter - - # Find the correct index for the answer key in the Arabic version - answer_index = valid_keys_latin.index(line["Answer Key"]) - - # Construct the query with Arabic letters - query = f"{instruction}{line['Question']}\n" - query += "".join([f"{key}. {choice}\n" for key, choice in zip(valid_keys_arabic, choices)]) - query += "الإجابة:" - - return Doc( - task_name=task_name, - query=query, - choices=valid_keys_arabic, # Return only valid choices (Arabic keys) - gold_index=answer_index, # Correct index in the valid Arabic keys - instruction=instruction, - ) - - -class CustomArabicMMLUTask(LightevalTaskConfig): - def __init__( - self, - name, - hf_subset, - ): - super().__init__( - name=name, - hf_subset=hf_subset, - prompt_function=arabic_mmlu_pfn, - hf_repo="MBZUAI/ArabicMMLU", - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=["dev"], - few_shots_select="sequential", - generation_size=-1, - stop_sequence=None, - version=0, - ) - - -ARABIC_MMLU_TASKS = [ - CustomArabicMMLUTask(name=f"arabic_mmlu:{subset}", hf_subset=subset) for subset in ARABIC_MMLU_SUBSETS -] - - -# ARABIC MMLU HT ## -# fmt: off -ARABIC_MMLU_HT_SUBSETS = [ - "abstract_algebra", "anatomy", "astronomy", "business_ethics", "clinical_knowledge", "college_biology", "college_chemistry", "college_computer_science", - "college_mathematics", "college_medicine", "college_physics", "computer_security", "conceptual_physics", "econometrics", "electrical_engineering", - "elementary_mathematics", "formal_logic", "global_facts", "high_school_biology", "high_school_chemistry", "high_school_computer_science", - "high_school_european_history", "high_school_geography", "high_school_government_and_politics", "high_school_macroeconomics", "high_school_mathematics", - "high_school_microeconomics", "high_school_physics", "high_school_psychology", "high_school_statistics", "high_school_us_history", "high_school_world_history", - "human_aging", "human_sexuality", "international_law", "jurisprudence", "logical_fallacies", "machine_learning", "management", "marketing", "medical_genetics", - "miscellaneous", "moral_disputes", "moral_scenarios", "nutrition", "philosophy", "prehistory", "professional_accounting", "professional_law", - "professional_medicine", "professional_psychology", "public_relations", "security_studies", "sociology", "us_foreign_policy", "virology", "world_religions" -] -# fmt: on - - -def arabic_mmlu_ht_pfn(line, task_name: str = None): - instruction = "السؤال التالي هو سؤال متعدد الإختيارات. اختر الإجابة الصحيحة:\n\n" - choices = line["choices"] - answer_index = line["answer"] # It is an int reflecting the index of correct answer in line["choices"] - - query = f"{instruction}{line['question']}\n" - query += "".join([f"{idx}. {choice}\n" for idx, choice in enumerate(choices, start=1)]) - query += "الإجابة:" - - return Doc( - task_name=task_name, - query=query, - choices=[str(i) for i in range(1, len(choices) + 1)], # List of strings instead of ints - gold_index=answer_index, - instruction=instruction, - ) - - -class CustomArabicMMLUHTTask(LightevalTaskConfig): - def __init__( - self, - name, - hf_subset, - ): - super().__init__( - name=name, - hf_subset=hf_subset, - prompt_function=arabic_mmlu_ht_pfn, - hf_repo="MBZUAI/human_translated_arabic_mmlu", - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - stop_sequence=None, - version=0, - ) - - -ARABIC_MMLU_HT_TASKS = [ - CustomArabicMMLUHTTask(name=f"arabic_mmlu_ht:{subset}", hf_subset=subset) for subset in ARABIC_MMLU_HT_SUBSETS -] - -# ARABIC MMLU MT ## -# fmt: off -ARABIC_MMLU_MT_SUBSETS = [ - "abstract_algebra", "anatomy", "astronomy", "business_ethics", "clinical_knowledge", "college_biology", "college_chemistry", "college_computer_science", - "college_mathematics", "college_medicine", "college_physics", "computer_security", "conceptual_physics", "econometrics", "electrical_engineering", - "elementary_mathematics", "formal_logic", "global_facts", "high_school_biology", "high_school_chemistry", "high_school_computer_science", - "high_school_european_history", "high_school_geography", "high_school_government_and_politics", "high_school_macroeconomics", "high_school_mathematics", - "high_school_microeconomics", "high_school_physics", "high_school_psychology", "high_school_statistics", "high_school_us_history", "high_school_world_history", - "human_aging", "human_sexuality", "international_law", "jurisprudence", "logical_fallacies", "machine_learning", "management", "marketing", "medical_genetics", - "miscellaneous", "moral_disputes", "moral_scenarios", "nutrition", "philosophy", "prehistory", "professional_accounting", "professional_law", - "professional_medicine", "professional_psychology", "public_relations", "security_studies", "sociology", "us_foreign_policy", "virology", "world_religions" -] -# fmt: on - - -def arabic_mmlu_mt_pfn(line, task_name: str = None): - instruction = "السؤال التالي هو سؤال متعدد الإختيارات. اختر الإجابة الصحيحة: أ، ب، ج، أو د... إلخ. \n\n" - choices = [line["A"], line["B"], line["C"], line["D"]] - # Answers are provided with roman letters - we look for the correct index in LETTER_INDICES, - # it will then be applied to arabic letters - answer_index = ascii_uppercase.index( - line["answer"] - ) # line["answer"] is the correct answer. That's why we need to index it ! - - query = f"{instruction}{line['question']}\n" - query += "".join([f"{key}. {choice}\n" for key, choice in zip(LETTER_INDICES_AR[:4], choices)]) - query += "الإجابة:" - - return Doc( - task_name=task_name, - query=query, - choices=LETTER_INDICES_AR[:4], - gold_index=answer_index, - instruction=instruction, - ) - - -class CustomArabicMMLUMTTask(LightevalTaskConfig): - def __init__( - self, - name, - hf_subset, - ): - super().__init__( - name=name, - hf_subset=hf_subset, - prompt_function=arabic_mmlu_mt_pfn, - hf_repo="OALL/Arabic_MMLU", - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], - hf_avail_splits=["test", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select="sequential", - generation_size=-1, - stop_sequence=None, - version=0, - ) - - -ARABIC_MMLU_MT_TASKS = [ - CustomArabicMMLUMTTask(name=f"arabic_mmlu_mt:{subset}", hf_subset=subset) for subset in ARABIC_MMLU_MT_SUBSETS -] - - -# ACVA ## -# fmt: off -ACVA_SUBSETS = [ - "Algeria", "Ancient_Egypt", "Arab_Empire", "Arabic_Architecture", "Arabic_Art", "Arabic_Astronomy", "Arabic_Calligraphy", "Arabic_Ceremony", - "Arabic_Clothing", "Arabic_Culture", "Arabic_Food", "Arabic_Funeral", "Arabic_Geography", "Arabic_History", "Arabic_Language_Origin", - "Arabic_Literature", "Arabic_Math", "Arabic_Medicine", "Arabic_Music", "Arabic_Ornament", "Arabic_Philosophy", "Arabic_Physics_and_Chemistry", - "Arabic_Wedding", "Bahrain", "Comoros", "Egypt_modern", "InfluenceFromAncientEgypt", "InfluenceFromByzantium", "InfluenceFromChina", - "InfluenceFromGreece", "InfluenceFromIslam", "InfluenceFromPersia", "InfluenceFromRome", "Iraq", "Islam_Education", "Islam_branches_and_schools", - "Islamic_law_system", "Jordan", "Kuwait", "Lebanon", "Libya", "Mauritania", "Mesopotamia_civilization", "Morocco", "Oman", "Palestine", "Qatar", - "Saudi_Arabia", "Somalia", "Sudan", "Syria", "Tunisia", "United_Arab_Emirates", "Yemen", - "communication", "computer_and_phone", "daily_life", "entertainment" -] -# fmt: on - - -def acva_pfn(line, task_name: str = None): - question = line["question"] - answer = line["answer"] - - return Doc( - task_name=task_name, - query=f"السؤال: {question}\nالإجابة:", - choices=["صح", "خطأ"], - gold_index=["صح", "خطأ"].index(answer), - ) - - -class CustomACVATask(LightevalTaskConfig): - def __init__( - self, - name, - hf_subset, - ): - super().__init__( - name=name, - hf_subset=hf_subset, - prompt_function=acva_pfn, - hf_repo="OALL/ACVA", - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], - hf_avail_splits=["test", "validation"], - evaluation_splits=["test"], - few_shots_split="validation", - few_shots_select="sequential", - generation_size=-1, - stop_sequence=None, - version=0, - ) - - -ACVA_TASKS = [CustomACVATask(name=f"acva:{subset}", hf_subset=subset) for subset in ACVA_SUBSETS] - - -# AraTrust ## -# fmt: off -ARATRUST_SUBSETS = [ - "Trustfulness", "MentalHealth", "PhysicalHealth", "Offensive", "Ethics", "Privacy", "Unfairness", "Illegal", -] -# fmt: on - - -def aratrust_pfn(line, task_name: str = None): - instruction = "السؤال التالي هو سؤال متعدد الإختيارات. اختر الإجابة الصحيحة: أ، ب أو ج. \n\n" - choices = [line["A"], line["B"], line["C"]] - # Answers are provided with roman letters - we look for the correct index in LETTER_INDICES, - # it will then be applied to arabic letters - answer_index = LETTER_INDICES_AR.index( - line["Answer"] - ) # line["answer"] is the correct answer. That's why we need to index it ! - - query = f"{instruction}{line['Question']}\n" - query += "".join([f"{choice}\n" for choice in choices]) - query += "الإجابة:" - - return Doc( - task_name=task_name, - query=query, - choices=LETTER_INDICES_AR[:3], - gold_index=answer_index, - instruction=instruction, - ) - - -class CustomAraTrustTask(LightevalTaskConfig): - def __init__( - self, - name, - hf_subset, - ): - super().__init__( - name=name, - hf_subset=hf_subset, - prompt_function=aratrust_pfn, - hf_repo="asas-ai/AraTrust-categorized", - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - stop_sequence=None, - version=0, - ) - - -ARATRUST_TASKS = [CustomAraTrustTask(name=f"aratrust:{subset}", hf_subset=subset) for subset in ARATRUST_SUBSETS] - - -def arabic_exams_pfn(line, task_name: str = None): - topic = line["subject"] - question = line["question"] - choices = [line["A"], line["B"], line["C"], line["D"]] - choices_formatted = [f" {LETTER_INDICES_AR[i]}) {choice}\n" for i, choice in enumerate(choices)] - answer = line["answer"] - answer_index = ascii_uppercase.index(answer) - - instruction = f"الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح حول {topic.replace('_', ' ')}. \n\n" - query = f"{instruction}السؤال: {question}\n" - query += "\n".join(choices_formatted) - query += "\nالإجابة:" - - return Doc( - task_name=task_name, - query=query, - choices=LETTER_INDICES_AR[:4], - gold_index=answer_index, - instruction=instruction, - ) - - -# ARABIC EXAMS ## -arabic_exams_task = LightevalTaskConfig( - name="arabic_exams", - prompt_function=arabic_exams_pfn, - hf_repo="OALL/Arabic_EXAMS", - hf_subset="default", - hf_avail_splits=["test", "validation"], - evaluation_splits=["test"], - few_shots_split="validation", - few_shots_select="sequential", - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], - version=0, -) - - -# ALGHAFA NATIVE ## -# fmt: off -ALGHAFA_SUBSETS = [ - "mcq_exams_test_ar", "meta_ar_dialects", "meta_ar_msa", "multiple_choice_facts_truefalse_balanced_task", "multiple_choice_grounded_statement_soqal_task", - "multiple_choice_grounded_statement_xglue_mlqa_task", "multiple_choice_rating_sentiment_no_neutral_task", "multiple_choice_rating_sentiment_task", - "multiple_choice_sentiment_task" -] -# fmt: on - - -def alghafa_pfn(line, task_name: str = None): - question = line["query"] - answer_index = int(line["label"]) - allowed_keys = [f"sol{i}" for i in range(1, 6)] - extracted_choices = [line[key] for key in allowed_keys if key in line] - choices = [str(i) for i in range(len(extracted_choices))] - - instruction = "الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح\n\n" - query = f"{instruction}السؤال: {question}\n" - - for index, choice in enumerate(extracted_choices): - query += f"{index}) {choice}\n" - - query += "الإجابة:" - - return Doc( - task_name=task_name, - query=query, - choices=choices, - gold_index=answer_index, - instruction=instruction, - ) - - -class CustomAlGhafaNativeTask(LightevalTaskConfig): - def __init__( - self, - name, - hf_subset, - ): - super().__init__( - name=name, - hf_subset=hf_subset, - prompt_function=alghafa_pfn, - hf_repo="OALL/AlGhafa-Arabic-LLM-Benchmark-Native", - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], - hf_avail_splits=["test", "validation"], - evaluation_splits=["test"], - few_shots_split="validation", - few_shots_select="sequential", - generation_size=-1, - stop_sequence=None, - version=0, - ) - - -ALGHAFA_TASKS = [CustomAlGhafaNativeTask(name=f"alghafa:{subset}", hf_subset=subset) for subset in ALGHAFA_SUBSETS] - -# ALGHAFA TRANSLATED ## -# race_ar -race_ar_task = LightevalTaskConfig( - name="race_ar", - prompt_function=alghafa_pfn, - hf_repo="OALL/AlGhafa-Arabic-LLM-Benchmark-Translated", - hf_subset="race_ar", - hf_avail_splits=["test", "validation"], - evaluation_splits=["test"], - few_shots_split="validation", - few_shots_select="sequential", - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], - version=0, -) - - -# piqa_ar -piqa_ar_task = LightevalTaskConfig( - name="piqa_ar", - prompt_function=alghafa_pfn, - hf_repo="OALL/AlGhafa-Arabic-LLM-Benchmark-Translated", - hf_subset="piqa_ar", - hf_avail_splits=["test", "validation"], - evaluation_splits=["test"], - few_shots_split="validation", - few_shots_select="sequential", - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], - version=0, -) - - -# arc_easy_ar -arc_easy_ar_task = LightevalTaskConfig( - name="arc_easy_ar", - prompt_function=alghafa_pfn, - hf_repo="OALL/AlGhafa-Arabic-LLM-Benchmark-Translated", - hf_subset="arc_easy_ar", - hf_avail_splits=["test", "validation"], - evaluation_splits=["test"], - few_shots_split="validation", - few_shots_select="sequential", - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], - version=0, -) - - -# arc_challenge_okapi_ar -arc_challenge_okapi_ar_task = LightevalTaskConfig( - name="arc_challenge_okapi_ar", - prompt_function=alghafa_pfn, - hf_repo="OALL/AlGhafa-Arabic-LLM-Benchmark-Translated", - hf_subset="arc_challenge_okapi_ar", - hf_avail_splits=["test", "validation"], - evaluation_splits=["test"], - few_shots_split="validation", - few_shots_select="sequential", - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], - version=0, -) - - -# mmlu_okapi_ar -mmlu_okapi_ar_task = LightevalTaskConfig( - name="mmlu_okapi_ar", - prompt_function=alghafa_pfn, - hf_repo="OALL/AlGhafa-Arabic-LLM-Benchmark-Translated", - hf_subset="mmlu_okapi_ar", - hf_avail_splits=["test", "validation"], - evaluation_splits=["test"], - few_shots_split="validation", - few_shots_select="sequential", - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], - version=0, -) - - -# openbook_qa_ext_ar -openbook_qa_ext_ar_task = LightevalTaskConfig( - name="openbook_qa_ext_ar", - prompt_function=alghafa_pfn, - hf_repo="OALL/AlGhafa-Arabic-LLM-Benchmark-Translated", - hf_subset="openbook_qa_ext_ar", - hf_avail_splits=["test", "validation"], - evaluation_splits=["test"], - few_shots_split="validation", - few_shots_select="sequential", - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], - version=0, -) - - -# boolq_ar -def boolq_arabic_pfn(line, task_name: str = None): - question = line["question"] - passage = line["passage"] - instruction = "بناء على المقطع التالي، أجب عن السؤال ب نعم أو لا" - query = f"""{instruction} - المقطع : - {passage} - السؤال: - {question} - الإجابة: - """ - - return Doc( - task_name=task_name, - query=query, - choices=["نعم", "لا"], - gold_index=0 if line["answer"] else 1, - instruction=instruction, - ) - - -boolq_ar_task = LightevalTaskConfig( - name="boolq_ar", - prompt_function=boolq_arabic_pfn, - hf_repo="OALL/AlGhafa-Arabic-LLM-Benchmark-Translated", - hf_subset="boolq_ar", - hf_avail_splits=["test", "validation"], - evaluation_splits=["test"], - few_shots_split="validation", - few_shots_select="sequential", - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], - version=0, -) - - -# copa_ext_ar -def copa_arabic_pfn(line, task_name: str = None): - premise = line["premise"] - choices = [line["choice1"], line["choice2"]] - question_map = {"cause": "لأن", "effect": "لذلك"} - question = question_map[line["question"]] - answer = line["label"] - - query = "{}، {} :\n0) {}\n1) {}\nالإجابة:".format(premise, question, choices[0], choices[1]) - - return Doc( - task_name=task_name, - query=query, - choices=choices, - gold_index=answer, - instruction="", - ) - - -copa_ext_ar_task = LightevalTaskConfig( - name="copa_ext_ar", - prompt_function=copa_arabic_pfn, - hf_repo="OALL/AlGhafa-Arabic-LLM-Benchmark-Translated", - hf_subset="copa_ext_ar", - hf_avail_splits=["test", "validation"], - evaluation_splits=["test"], - few_shots_split="validation", - few_shots_select="sequential", - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], - version=0, -) - - -# hellaswag_okapi_ar -def hellaswag_arabic_pfn(line, task_name: str = None): - ctx = re.sub(r"\[.*?\]", "", line["ctx"]) # Remove latin words within brackets - endings = [ - re.sub(r"\[.*?\]", "", e) for e in eval(line["endings"]) - ] # endings is a string representation of a list - answer_index = line["label"] - instruction = "بناء على السياق التالي، اختر النهاية الصحيحة من الاقتراحات التالية" - - query = f"""{instruction} - السياق: - {ctx} - الاقتراحات: - - """ - for i, ending in enumerate(endings): - query += f"{i}) {ending}\n" - query += "الإجابة:" - - return Doc( - task_name=task_name, - query=query, - choices=endings, - gold_index=answer_index, - instruction=instruction, - ) - - -hellaswag_okapi_ar_task = LightevalTaskConfig( - name="hellaswag_okapi_ar", - prompt_function=hellaswag_arabic_pfn, - hf_repo="OALL/AlGhafa-Arabic-LLM-Benchmark-Translated", - hf_subset="hellaswag_okapi_ar", - hf_avail_splits=["test", "validation"], - evaluation_splits=["test"], - few_shots_split="validation", - few_shots_select="sequential", - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], - version=0, -) - - -# toxigen_ar -def toxigen_arabic_pfn(line, task_name: str = None): - text = line["text"] - label = 1 if ((line["toxicity_ai"] + line["toxicity_human"]) > 5.5) else 0 - instruction = 'هل العبارة التالية تحض على الكراهية؟ قم بالرد بـ"نعم" أو "لا".' - - query = f"""{instruction} - العبارة: - '{text}' - الإجابة: - """ - - return Doc( - task_name=task_name, - query=query, - choices=["لا", "نعم"], - gold_index=label, - instruction=instruction, - ) - - -toxigen_ar_task = LightevalTaskConfig( - name="toxigen_ar", - prompt_function=toxigen_arabic_pfn, - hf_repo="OALL/AlGhafa-Arabic-LLM-Benchmark-Translated", - hf_subset="toxigen_ar", - hf_avail_splits=["test", "validation"], - evaluation_splits=["test"], - few_shots_split="validation", - few_shots_select="sequential", - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], - version=0, -) - - -# sciq_ar -def sciq_arabic_pfn(line, task_name: str = None): - support = line["support"] - question = line["question"] - correct_answer = line["correct_answer"] - choices = [line["distractor1"], line["distractor2"], line["distractor3"], correct_answer] - - # Shuffle the choices - random.shuffle(choices) - - answer_index = choices.index(correct_answer) - - instruction = "بناءً على السياق أدناه، اختر الإجابة الصحيحة للسؤال التالي من قائمة الاقتراحات" - - query = f"""{instruction} - السياق: - {support} - السؤال: - {question} - الإجابات المحتملة: - - """ - for i, choice in enumerate(choices): - query += f"{i}) {choice}\n" - query += "الإجابة:" - - return Doc( - task_name=task_name, - query=query, - choices=choices, - gold_index=answer_index, - instruction=instruction, - ) - - -sciq_ar_task = LightevalTaskConfig( - name="sciq_ar", - prompt_function=sciq_arabic_pfn, - hf_repo="OALL/AlGhafa-Arabic-LLM-Benchmark-Translated", - hf_subset="sciq_ar", - hf_avail_splits=["test", "validation"], - evaluation_splits=["test"], - few_shots_split="validation", - few_shots_select="sequential", - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], - version=0, -) - - -# madinah_qa -# fmt: off -MADINAH_QA_SUBSETS = ["Arabic Language (General)", "Arabic Language (Grammar)"] -# fmt: on - - -def madinah_qa_pfn(line, task_name: str = None): - instruction = "بناءً على السياق أدناه، اختر الإجابة الصحيحة للسؤال التالي من قائمة الأجوبة:\n\n" - - # Define the mapping from Latin to Arabic letters - latin_to_arabic = {"A": "أ", "B": "ب", "C": "ج", "D": "د", "E": "هـ"} - - # Create a list of valid choices with corresponding Arabic keys - choices = [] - valid_keys_latin = [] - valid_keys_arabic = [] - - # Enumerate through the options and append the valid ones - for idx, key in enumerate(["A", "B", "C", "D", "E"]): - option = line.get(f"Option {idx + 1}") - if option: # Check if option is not null - choices.append(option) - valid_keys_latin.append(key) # Append the Latin key (A, B, C, D, E) - valid_keys_arabic.append(latin_to_arabic[key]) # Append the corresponding Arabic letter - - # Find the correct index for the answer key in the Arabic version - answer_index = valid_keys_latin.index(line["Answer Key"]) - - query = f"{instruction}\nالسياق:\n{line['Context']}\nالسؤال:\n{line['Question']}\n" - query += "".join([f"{key}. {choice}\n" for key, choice in zip(valid_keys_arabic, choices)]) - query += "الإجابة:" - - return Doc( - task_name=task_name, - query=query, - choices=valid_keys_arabic, - gold_index=answer_index, # Correct index in the valid keys - instruction=instruction, - ) - - -class CustomMadinahQATask(LightevalTaskConfig): - def __init__( - self, - name, - hf_subset, - ): - super().__init__( - name=name, - hf_subset=hf_subset, - prompt_function=madinah_qa_pfn, - hf_repo="MBZUAI/MadinahQA", - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=["dev"], - few_shots_select="sequential", - generation_size=-1, - stop_sequence=None, - version=0, - ) - - -MADINAH_QA_TASKS = [ - CustomMadinahQATask(name=f"madinah_qa:{subset}", hf_subset=subset) for subset in MADINAH_QA_SUBSETS -] - - -class JudgeMetricWrapper(Metric): - """Wrapper class for LLM-based judge metric implementation.""" - - def __init__(self, judge: JudgeLM): - """ - Initializes the judge metric wrapper. - - Args: - judge (JudgeLM): The LLM judge instance to use for evaluation. - """ - self.judge = judge - self.metric_name = "llm_as_judge" - self.category = SamplingMethod.GENERATIVE - self.corpus_level_fn = self.aggregate_scores - self.sample_level_fn = self._sample_level_fn - self.higher_is_better = True # Fixed tuple syntax - - def compute(self, responses: list[str], formatted_docs: list[Doc], **kwargs) -> dict[str, float]: - """ - Computes evaluation scores using the judge's evaluate_answer method. - - Args: - responses (list[str]): The predicted answers - formatted_docs (list[Doc]): Documents containing questions and gold answers - kwargs: Additional keyword arguments (not used) - - Returns: - dict[str, float]: Dictionary containing evaluation scores - """ - results = [] - for i, doc in enumerate(formatted_docs): - question = doc.query - gold = doc.choices[doc.gold_index] if doc.gold_index is not None else None - answer = responses[i][0].result[0] - - score, _, _ = self.judge.evaluate_answer(question=question, answer=answer, options=None, gold=gold) - results.append({self.metric_name: score}) - - return results - - def aggregate_scores(self, scores: list[dict]) -> float: - return sum(scores) / len(scores) if scores else 0.0 - - def _sample_level_fn(self): - return None - - -def parse_candidates(candidates: Union[List[str], str]) -> List[str]: - """ - Parses and validates candidate answers from either list or string format. - - Args: - candidates: Either a list of candidate answers or a newline-separated string - - Returns: - List[str]: List of validated candidate answers - - Raises: - ValueError: If candidates cannot be parsed or are empty - """ - try: - if isinstance(candidates, list): - parsed_candidates = [str(c).strip() for c in candidates if c] - else: - parsed_candidates = [c.strip() for c in str(candidates).split("\n") if c.strip()] - - if not parsed_candidates: - raise ValueError("No valid candidates found after parsing") - - return parsed_candidates - except Exception as e: - raise ValueError(f"Failed to parse candidates: {str(e)}") - - -def qa_prompt_arabic(line: Dict[str, Any], task_name: str = None) -> Doc: - """ - Formats the prompt for Arabic question answering with candidates. - - Args: - line: Dictionary containing question and candidate information - task_name: Optional name for the task - - Returns: - Doc: Formatted document for evaluation - - Raises: - ValueError: If required fields are missing or invalid - """ - try: - # Validates and extracts the question - if not isinstance(line.get("question"), str): - raise ValueError("Question must be a string") - question = line["question"] - - # Processes candidate answers - candidates = parse_candidates(line["candidates"]) - - # Validates gold answer - if "gold_answer" not in line: - raise ValueError("Gold answer is required") - gold_answer = str(line["gold_answer"]) - - # Constructs the prompt - instruction = "بناءً على السياقات المقترحة التالية، اجب عن السؤال التالي" - query = f"{instruction}\n\nالسؤال:\n{question}\n\nالسياقات المقترحة:\n{', '.join(candidates)}\n" - - return Doc( - task_name=task_name or "alrage", - query=query, - instruction=instruction, - choices=[gold_answer], # Gold answer is used as the only valid choice - gold_index=0, # Index of the correct answer in choices - ) - except Exception as e: - raise ValueError(f"Failed to create QA prompt: {str(e)}") - - -def judge_template(question: str, answer: str, gold: str, options: Optional[List[str]] = None) -> List[Dict[str, str]]: - """ - Template for the Arabic judge prompt. - - System prompt translation: - You are a neutral expert evaluator. Your tasks are: - 1. Evaluate the answer's accuracy compared to the correct answer - 2. Verify that the answer is supported by the provided context - 3. Evaluate the quality and comprehensiveness of the answer - Rate the answer on a scale from 0 to 10. - - Args: - question: The question being evaluated - answer: The provided answer - gold: The correct answer - options: Optional list of answer choices - - Returns: - List[Dict[str, str]]: Formatted messages for the judge - """ - messages = [ - { - "role": "system", - "content": """أنت مقيّم محايد خبير باللغة العربية. يجب عليك: -1. تقييم دقة الإجابة مقارنة بالإجابة الصحيحة -2. التحقق من أن الإجابة مدعومة بالسياق المقدم -3. تقييم جودة وشمولية الإجابة - -مهم جداً: يجب أن يكون ردك رقماً فقط من 0 إلى 10. لا تضف أي نص أو تفسير.""", - }, - { - "role": "user", - "content": f"""السؤال: {question} - -الإجابة المقدمة: {answer} - -الإجابة الصحيحة: {gold} - -أعط تقييماً من 0 إلى 10: -0-2: إجابة خاطئة تماماً -3-4: إجابة جزئية مع أخطاء -5-6: إجابة متوسطة -7-8: إجابة جيدة -9-10: إجابة ممتازة - -اكتب رقماً فقط من 0 إلى 10 بدون أي نص إضافي:""", - }, - ] - return messages - - -def process_judge_response(response) -> float: - """Process the judge's response to extract the score""" - # If response is a list, extract the content from the user role - if isinstance(response, list): - response_content = " ".join(item["content"] for item in response if item["role"] == "user") - else: - response_content = response # If it's not a list, use it directly - - try: - # Extract the score from the response content - score = float(next(num for num in response_content.split() if num.replace(".", "", 1).isdigit())) - return min(max(score / 10.0, 0.0), 1.0) - except (StopIteration, ValueError): - return 0.0 - - -judge = JudgeLM( - model="Qwen/Qwen2.5-72B-Instruct", - templates=judge_template, - process_judge_response=process_judge_response, - judge_backend="vllm", -) - -wrapped_judge = JudgeMetricWrapper(judge) - -# Task configuration -alrage_qa_task = LightevalTaskConfig( - name="alrage_qa", - prompt_function=qa_prompt_arabic, - hf_repo="OALL/ALRAGE", - hf_subset=None, - hf_avail_splits=["train"], - evaluation_splits=["train"], - metrics=[wrapped_judge], - generation_size=200, - stop_sequence=[], - version=0, -) - -TASKS_TABLE = ( - ARABIC_MMLU_TASKS - + ARABIC_MMLU_HT_TASKS - + ARABIC_MMLU_MT_TASKS - + ACVA_TASKS - + ALGHAFA_TASKS - + ARATRUST_TASKS - + MADINAH_QA_TASKS - + [arabic_exams_task] - + [race_ar_task] - + [piqa_ar_task] - + [arc_easy_ar_task] - + [arc_challenge_okapi_ar_task] - + [mmlu_okapi_ar_task] - + [openbook_qa_ext_ar_task] - + [boolq_ar_task] - + [copa_ext_ar_task] - + [hellaswag_okapi_ar_task] - + [toxigen_ar_task] - + [sciq_ar_task] - + [alrage_qa_task] -) diff --git a/src/lighteval/tasks/multilingual/tasks/arabic_arc.py b/src/lighteval/tasks/multilingual/tasks/arabic_arc.py deleted file mode 100644 index ab1fef6f2..000000000 --- a/src/lighteval/tasks/multilingual/tasks/arabic_arc.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -name: -Arabic Arc - -dataset: -OALL/AlGhafa-Arabic-LLM-Benchmark-Translated - -abstract: -Arabic Arc multilingual benchmark. - -languages: -arabic - -tags: -multilingual, multiple-choice, reasoning - -paper: -""" - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.adapters import ( - alghafa_adapter, -) -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"alghafa_arc_{Language.ARABIC.value}_{formulation.name.lower()}:easy", - prompt_function=get_mcq_prompt_function(Language.ARABIC, alghafa_adapter, formulation=formulation), - hf_repo="OALL/AlGhafa-Arabic-LLM-Benchmark-Translated", - hf_revision="08663706ee7cab30c4b7dc1bb00042a3227ce1ff", - hf_subset="arc_easy_ar", - evaluation_splits=["test"], - few_shots_split="validation", - few_shots_select="sequential", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/arabic_mmlu.py b/src/lighteval/tasks/multilingual/tasks/arabic_mmlu.py deleted file mode 100644 index db7010522..000000000 --- a/src/lighteval/tasks/multilingual/tasks/arabic_mmlu.py +++ /dev/null @@ -1,113 +0,0 @@ -""" -name: -Arabic Mmlu - -dataset: -MBZUAI/ArabicMMLU - -abstract: -Arabic Mmlu multilingual benchmark. - -languages: -arabic - -tags: -knowledge, multilingual, multiple-choice - -paper: -""" - -from string import ascii_uppercase - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbPMINorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation, normalize_subset -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -ARABIC_MMLU_SUBSETS = [ - "Islamic Studies", - "Islamic Studies (Middle School)", - "Islamic Studies (Primary School)", - "Islamic Studies (High School)", - "Driving Test", - "Natural Science (Middle School)", - "Natural Science (Primary School)", - "History (Middle School)", - "History (Primary School)", - "History (High School)", - "General Knowledge", - "General Knowledge (Middle School)", - "General Knowledge (Primary School)", - "Law (Professional)", - "Physics (High School)", - "Social Science (Middle School)", - "Social Science (Primary School)", - "Management (University)", - "Arabic Language (Middle School)", - "Arabic Language (Primary School)", - "Arabic Language (High School)", - "Political Science (University)", - "Philosophy (High School)", - "Accounting (University)", - "Computer Science (Middle School)", - "Computer Science (Primary School)", - "Computer Science (High School)", - "Computer Science (University)", - "Geography (Middle School)", - "Geography (Primary School)", - "Geography (High School)", - "Math (Primary School)", - "Biology (High School)", - "Economics (Middle School)", - "Economics (High School)", - "Economics (University)", - "Arabic Language (General)", - "Arabic Language (Grammar)", - "Civics (Middle School)", - "Civics (High School)", -] - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"mmlu_{Language.ARABIC.value}_{formulation.name.lower()}:{normalize_subset(subset)}", - prompt_function=get_mcq_prompt_function( - Language.ARABIC, - lambda line: { - "context": line["Context"], - "question": line["Question"], - "choices": [str(o) for o in [line[f"Option {i}"] for i in range(1, 6)] if o], - "gold_idx": ascii_uppercase.index(line["Answer Key"]), - }, - formulation=formulation, - ), - hf_repo="MBZUAI/ArabicMMLU", - hf_subset=subset, - evaluation_splits=("test",), - hf_avail_splits=["dev"], - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbPMINorm()), - ], - ), - ) - for subset in ARABIC_MMLU_SUBSETS - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/arcd.py b/src/lighteval/tasks/multilingual/tasks/arcd.py deleted file mode 100644 index abb804775..000000000 --- a/src/lighteval/tasks/multilingual/tasks/arcd.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -name: -Arcd - -dataset: -hsseinmz/arcd - -abstract: -ARCD: Arabic Reading Comprehension Dataset. - -languages: -arabic - -tags: -multilingual, multiple-choice, qa, reasoning - -paper: -https://arxiv.org/pdf/1906.05394 -""" - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -# ARCD: Arabic Reading Comprehension Dataset. -# https://arxiv.org/pdf/1906.05394 - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"arcd_{Language.ARABIC.value}", - prompt_function=get_qa_prompt_function( - Language.ARABIC, - lambda line: { - "question": line["question"], - "context": line["context"], - "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0], - }, - ), - hf_repo="hsseinmz/arcd", - hf_subset="plain_text", - evaluation_splits=("validation",), - few_shots_split="train", - metrics=( - MultilingualQuasiExactMatchMetric(Language.ARABIC, "prefix"), - MultilingualQuasiF1ScoreMetric(Language.ARABIC), - ), - generation_size=400, - stop_sequence=("\n",), - ) -] diff --git a/src/lighteval/tasks/multilingual/tasks/belebele.py b/src/lighteval/tasks/multilingual/tasks/belebele.py deleted file mode 100644 index 5543a9207..000000000 --- a/src/lighteval/tasks/multilingual/tasks/belebele.py +++ /dev/null @@ -1,191 +0,0 @@ -""" -name: -Belebele - -dataset: -facebook/belebele - -abstract: -Belebele: A large-scale reading comprehension dataset covering 122 languages. - -languages: -arabic, armenian, bengali, cyrillic, devanagari, ethiopic, georgian, greek, -gujarati, gurmukhi, chinese (simplified), chinese (traditional), hangul, hebrew, -japanese, khmer, kannada, lao, latin, malayalam, myanmar, odia, sinhala, tamil, -telugu, thai, tibetan - -tags: -multilingual, multiple-choice, reading-comprehension - -paper: -https://arxiv.org/abs/2308.16884 -""" - -from langcodes import Language as LangCodeLanguage - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import iso_639_3_ind_to_iso_639_3_macro - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"belebele_{language}_{formulation.name.lower()}", - prompt_function=get_mcq_prompt_function( - iso_639_3_ind_to_iso_639_3_macro[LangCodeLanguage.get(language).to_alpha3()], - lambda line: { - "question": line["question"], - "context": line["flores_passage"], - "choices": [line[f"mc_answer{i}"] for i in range(1, 5)], - "gold_idx": int(line["correct_answer_num"]) - 1, - }, - formulation=formulation, - ), - hf_repo="facebook/belebele", - hf_subset=language, - evaluation_splits=("test",), - hf_avail_splits=["test"], - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] - for language in [ - "acm_Arab", - "arz_Arab", - "ceb_Latn", - "fin_Latn", - "hin_Deva", - "ita_Latn", - "khm_Khmr", - "lvs_Latn", - "npi_Deva", - "pol_Latn", - "slv_Latn", - "swe_Latn", - # "tso_Latn", - # "xho_Latn", - "afr_Latn", - "asm_Beng", - "ces_Latn", - "fra_Latn", - "hin_Latn", - "jav_Latn", - # "kin_Latn", - "mal_Mlym", - "npi_Latn", - "por_Latn", - # "sna_Latn", - "swh_Latn", - "tur_Latn", - "yor_Latn", - "als_Latn", - "azj_Latn", - "ckb_Arab", - # "fuv_Latn", - "hrv_Latn", - "jpn_Jpan", - "kir_Cyrl", - "mar_Deva", - # "nso_Latn", - "snd_Arab", - "tam_Taml", - "ukr_Cyrl", - "zho_Hans", - "amh_Ethi", - # "bam_Latn", - "dan_Latn", - # "gaz_Latn", - "hun_Latn", - # "kac_Latn", - "kor_Hang", - "mkd_Cyrl", - # "nya_Latn", - "ron_Latn", - "som_Latn", - "tel_Telu", - "urd_Arab", - "zho_Hant", - "apc_Arab", - "ben_Beng", - "deu_Latn", - # "grn_Latn", - "hye_Armn", - "kan_Knda", - "lao_Laoo", - "mlt_Latn", - "ory_Orya", - "rus_Cyrl", - # "sot_Latn", - "tgk_Cyrl", - "urd_Latn", - "zsm_Latn", - "arb_Arab", - "ben_Latn", - "ell_Grek", - "guj_Gujr", - # "ibo_Latn", - "kat_Geor", - # "lin_Latn", - # "mri_Latn", - "pan_Guru", - # "shn_Mymr", - "spa_Latn", - "tgl_Latn", - "uzn_Latn", - # "zul_Latn", - "arb_Latn", - # "bod_Tibt", - "eng_Latn", - # "hat_Latn", - # "ilo_Latn", - "kaz_Cyrl", - "lit_Latn", - "mya_Mymr", - "pbt_Arab", - "sin_Latn", - "srp_Cyrl", - "tha_Thai", - "vie_Latn", - "ars_Arab", - "bul_Cyrl", - "est_Latn", - # "hau_Latn", - "ind_Latn", - # "kea_Latn", - # "lug_Latn", - "nld_Latn", - "pes_Arab", - "sin_Sinh", - # "ssw_Latn", - # "tir_Ethi", - "war_Latn", - "ary_Arab", - "cat_Latn", - "eus_Latn", - "heb_Hebr", - "isl_Latn", - # "khk_Cyrl", - # "luo_Latn", - "nob_Latn", - "plt_Latn", - "slk_Latn", - # "sun_Latn", - # "tsn_Latn", - # "wol_Latn", - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/c3.py b/src/lighteval/tasks/multilingual/tasks/c3.py deleted file mode 100644 index 8e0547f34..000000000 --- a/src/lighteval/tasks/multilingual/tasks/c3.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -name: -C3 - -dataset: -clue/clue - -abstract: -C3: A Chinese Challenge Corpus for Cross-lingual and Cross-modal Tasks Reading -comprehension task part of clue. - -languages: -chinese - -tags: -multilingual, multiple-choice, reasoning - -paper: -https://arxiv.org/abs/2004.05986 -""" - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -# C3: A Chinese Challenge Corpus for Cross-lingual and Cross-modal Tasks -# Reading comprehension task part of clue -# Paper: https://arxiv.org/abs/2004.05986 - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"c3_{Language.CHINESE.value}_{formulation.name.lower()}", - prompt_function=get_mcq_prompt_function( - Language.CHINESE, - lambda line: { - "question": line["question"], - "choices": line["choice"], - "gold_idx": line["choice"].index(line["answer"]), - "context": " ".join(line["context"]), - }, - formulation=formulation, - ), - hf_repo="clue/clue", - hf_subset="c3", - evaluation_splits=("validation",), - few_shots_split="train", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/ceval.py b/src/lighteval/tasks/multilingual/tasks/ceval.py deleted file mode 100644 index 51ab76c22..000000000 --- a/src/lighteval/tasks/multilingual/tasks/ceval.py +++ /dev/null @@ -1,126 +0,0 @@ -""" -name: -Ceval - -dataset: -ceval/ceval-exam - -abstract: -Ceval multilingual benchmark. - -languages: -chinese - -tags: -knowledge, multilingual, multiple-choice - -paper: -""" - -from functools import partial - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.adapters import ( - ceval_adapter, -) -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -CEVAL_SUBSET = [ - "computer_network", - "operating_system", - "computer_architecture", - "college_programming", - "college_physics", - "college_chemistry", - "advanced_mathematics", - "probability_and_statistics", - "discrete_mathematics", - "electrical_engineer", - "metrology_engineer", - "high_school_mathematics", - "high_school_physics", - "high_school_chemistry", - "high_school_biology", - "middle_school_mathematics", - "middle_school_biology", - "middle_school_physics", - "middle_school_chemistry", - "veterinary_medicine", - "college_economics", - "business_administration", - "marxism", - "mao_zedong_thought", - "education_science", - "teacher_qualification", - "high_school_politics", - "high_school_geography", - "middle_school_politics", - "middle_school_geography", - "modern_chinese_history", - "ideological_and_moral_cultivation", - "logic", - "law", - "chinese_language_and_literature", - "art_studies", - "professional_tour_guide", - "legal_professional", - "high_school_chinese", - "high_school_history", - "middle_school_history", - "civil_servant", - "sports_science", - "plant_protection", - "basic_medicine", - "clinical_medicine", - "urban_and_rural_planner", - "accountant", - "fire_engineer", - "environmental_impact_assessment_engineer", - "tax_accountant", - "physician", -] - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"ceval_{Language.CHINESE.value}_{formulation.name.lower()}:{subset}", - prompt_function=get_mcq_prompt_function( - Language.CHINESE, - partial( - ceval_adapter, - Language.CHINESE, - formulation, - ), - formulation=formulation, - ), - hf_repo="ceval/ceval-exam", - hf_subset=subset, - evaluation_splits=("val",), - few_shots_split="dev", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for subset in CEVAL_SUBSET - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/chegeka.py b/src/lighteval/tasks/multilingual/tasks/chegeka.py deleted file mode 100644 index 05207a463..000000000 --- a/src/lighteval/tasks/multilingual/tasks/chegeka.py +++ /dev/null @@ -1,50 +0,0 @@ -""" -name: -Chegeka - -dataset: -ai-forever/MERA - -abstract: -Chegeka multilingual benchmark. - -languages: -russian - -tags: -knowledge, multilingual, qa - -paper: -""" - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"chegeka_{Language.RUSSIAN.value}", - prompt_function=get_qa_prompt_function( - Language.RUSSIAN, - lambda line: { - "question": line["inputs"]["text"], - "choices": [line["outputs"]], - }, - ), - hf_repo="ai-forever/MERA", - hf_subset="chegeka", - evaluation_splits=("train",), - hf_avail_splits=["train"], - generation_size=400, - stop_sequence=("\n",), - metrics=[ - MultilingualQuasiExactMatchMetric(Language.RUSSIAN, "prefix"), - MultilingualQuasiF1ScoreMetric(Language.RUSSIAN), - ], - ) -] diff --git a/src/lighteval/tasks/multilingual/tasks/chinese_squad.py b/src/lighteval/tasks/multilingual/tasks/chinese_squad.py deleted file mode 100644 index 8fae8c4dc..000000000 --- a/src/lighteval/tasks/multilingual/tasks/chinese_squad.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -name: -Chinese Squad - -dataset: -lighteval/ChineseSquad - -abstract: -ChineseSquad is a reading comprehension dataset for Chinese. - -languages: -chinese - -tags: -multilingual, qa - -paper: -https://github.com/pluto-junzeng/ChineseSquad -""" - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"chinese_squad_{Language.CHINESE.value}", - prompt_function=get_qa_prompt_function( - Language.CHINESE, - lambda line: { - "question": line["question"], - "context": line["context"], - "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0], - }, - ), - hf_repo="lighteval/ChineseSquad", - hf_subset="default", - evaluation_splits=("validation",), - few_shots_split="train", - metrics=( - MultilingualQuasiExactMatchMetric(Language.CHINESE, "prefix"), - MultilingualQuasiF1ScoreMetric(Language.CHINESE), - ), - generation_size=400, - stop_sequence=("\n",), - ) -] diff --git a/src/lighteval/tasks/multilingual/tasks/cmath.py b/src/lighteval/tasks/multilingual/tasks/cmath.py deleted file mode 100644 index 7ce917d73..000000000 --- a/src/lighteval/tasks/multilingual/tasks/cmath.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -name: -Cmath - -dataset: -weitianwen/cmath - -abstract: -Cmath multilingual benchmark. - -languages: -chinese - -tags: -math, multilingual, reasoning - -paper: -""" - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"cmath_{Language.CHINESE.value}", - prompt_function=get_qa_prompt_function( - Language.CHINESE, - lambda line: { - "question": line["question"], - "choices": [line["golden"]], - }, - ), - hf_repo="weitianwen/cmath", - hf_subset="default", - evaluation_splits=("test",), - few_shots_split="validation", - generation_size=25, - metrics=[ - MultilingualQuasiExactMatchMetric(Language.CHINESE, "full"), - ], - stop_sequence=("\n",), - ) -] diff --git a/src/lighteval/tasks/multilingual/tasks/cmmlu.py b/src/lighteval/tasks/multilingual/tasks/cmmlu.py deleted file mode 100644 index 566fad0f2..000000000 --- a/src/lighteval/tasks/multilingual/tasks/cmmlu.py +++ /dev/null @@ -1,139 +0,0 @@ -""" -name: -Cmmlu - -dataset: -haonan-li/cmmlu - -abstract: -Cmmlu multilingual benchmark. - -languages: -chinese - -tags: -knowledge, multilingual, multiple-choice - -paper: -""" - -from string import ascii_uppercase - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbPMINorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -CMMLU_SUBSETS = [ - "agronomy", - "anatomy", - "ancient_chinese", - "arts", - "astronomy", - "business_ethics", - "chinese_civil_service_exam", - "chinese_driving_rule", - "chinese_food_culture", - "chinese_foreign_policy", - "chinese_history", - "chinese_literature", - "chinese_teacher_qualification", - "clinical_knowledge", - "college_actuarial_science", - "college_education", - "college_engineering_hydrology", - "college_law", - "college_mathematics", - "college_medical_statistics", - "college_medicine", - "computer_science", - "computer_security", - "conceptual_physics", - "construction_project_management", - "economics", - "education", - "electrical_engineering", - "elementary_chinese", - "elementary_commonsense", - "elementary_information_and_technology", - "elementary_mathematics", - "ethnology", - "food_science", - "genetics", - "global_facts", - "high_school_biology", - "high_school_chemistry", - "high_school_geography", - "high_school_mathematics", - "high_school_physics", - "high_school_politics", - "human_sexuality", - "international_law", - "journalism", - "jurisprudence", - "legal_and_moral_basis", - "logical", - "machine_learning", - "management", - "marketing", - "marxist_theory", - "modern_chinese", - "nutrition", - "philosophy", - "professional_accounting", - "professional_law", - "professional_medicine", - "professional_psychology", - "public_relations", - "security_study", - "sociology", - "sports_science", - "traditional_chinese_medicine", - "virology", - "world_history", - "world_religions", -] - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"cmmlu_{Language.CHINESE.value}_{formulation.name.lower()}:{subset}", - prompt_function=get_mcq_prompt_function( - Language.CHINESE, - lambda line: { - "question": line["Question"], - "choices": [line["A"], line["B"], line["C"], line["D"]], - "gold_idx": ascii_uppercase.index(line["Answer"]), - }, - formulation=formulation, - ), - hf_repo="haonan-li/cmmlu", - hf_subset=subset, - evaluation_splits=("test",), - few_shots_split="dev", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbPMINorm()), - ], - ), - ) - for subset in CMMLU_SUBSETS - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/cmnli.py b/src/lighteval/tasks/multilingual/tasks/cmnli.py deleted file mode 100644 index bb4f95ab6..000000000 --- a/src/lighteval/tasks/multilingual/tasks/cmnli.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -name: -Cmnli - -dataset: -fenffef/cmnli - -abstract: -Native Chinese NLI dataset based on MNLI approach (Machine Translated) - -languages: -chinese - -tags: -classification, multilingual, nli - -paper: -https://arxiv.org/abs/2004.05986 -""" - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.nli import get_nli_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"cmnli_{Language.CHINESE.value}_{formulation.name.lower()}", - prompt_function=get_nli_prompt_function( - language=Language.CHINESE, - adapter=lambda line: { - "premise": line["sentence1"], - "hypothesis": line["sentence2"], - # Since we ignore the neutral label - "gold_idx": {"entailment": 0, "contradiction": 1}[line["label"]], - }, - relations=["entailment", "contradiction"], - formulation=formulation, - ), - hf_repo="fenffef/cmnli", - hf_subset="default", - hf_filter=lambda x: x["label"] in ["entailment", "contradiction"], - # Only keep the positive and negative examples - evaluation_splits=("validation",), - few_shots_split="train", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=None), - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] -] diff --git a/src/lighteval/tasks/multilingual/tasks/cmrc2018.py b/src/lighteval/tasks/multilingual/tasks/cmrc2018.py deleted file mode 100644 index c13154342..000000000 --- a/src/lighteval/tasks/multilingual/tasks/cmrc2018.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -name: -Cmrc2018 - -dataset: -clue/clue - -abstract: -CMRC 2018: A span-extraction machine reading comprehension dataset for Chinese. - -languages: -chinese - -tags: -multilingual, qa - -paper: -https://arxiv.org/abs/1810.07366 -""" - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"cmrc2018_{Language.CHINESE.value}", - prompt_function=get_qa_prompt_function( - Language.CHINESE, - lambda line: { - "question": line["question"], - "context": line["context"], - "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0], - }, - ), - hf_repo="clue/clue", - hf_subset="cmrc2018", - evaluation_splits=("trial",), - few_shots_split="train", - generation_size=400, - metrics=( - MultilingualQuasiExactMatchMetric(Language.CHINESE, "prefix"), - MultilingualQuasiF1ScoreMetric(Language.CHINESE), - ), - stop_sequence=("\n",), - ) -] diff --git a/src/lighteval/tasks/multilingual/tasks/copa_indic.py b/src/lighteval/tasks/multilingual/tasks/copa_indic.py deleted file mode 100644 index 8882b00da..000000000 --- a/src/lighteval/tasks/multilingual/tasks/copa_indic.py +++ /dev/null @@ -1,92 +0,0 @@ -""" -name: -Copa Indic - -dataset: -ai4bharat/IndicCOPA - -abstract: -IndicCOPA: COPA for Indic Languages Paper: https://arxiv.org/pdf/2212.05409 -IndicCOPA extends COPA to 15 Indic languages, providing a valuable resource for -evaluating common sense reasoning in these languages. - -languages: -assamese, bengali, gujarati, hindi, kannada, malayalam, marathi, nepali, oriya, -punjabi, sanskrit, sindhi, tamil, telugu, urdu - -tags: -multilingual, multiple-choice, reasoning - -paper: -https://arxiv.org/pdf/2212.05409 -""" - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.copa import get_copa_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -# IndicCOPA: COPA for Indic Languages -# Paper: https://arxiv.org/pdf/2212.05409 -# IndicCOPA extends COPA to 15 Indic languages, providing a valuable resource for -# evaluating common sense reasoning in these languages. - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"indicxcopa_{language.value}_{formulation.name.lower()}", - prompt_function=get_copa_prompt_function( - language, - adapter=lambda line: { - "context": line["premise"], - "cause_effect": line["question"], - "continuations": [line["choice1"], line["choice2"]], - "gold_idx": int(line["label"]), - }, - formulation=formulation, - ), - hf_repo="ai4bharat/IndicCOPA", - hf_subset=f"translation-{standardize_tag(language.value)}", - hf_revision="d356ef19a4eb287e88a51d07a56b73ba88c7f188", - evaluation_splits=["test"], - hf_avail_splits=["test"], - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for language in [ - Language.ASSAMESE, - Language.BENGALI, - Language.GUJARATI, - Language.HINDI, - Language.KANNADA, - Language.MALAYALAM, - Language.MARATHI, - Language.NEPALI, - Language.ORIYA, - Language.PUNJABI, - Language.SANSKRIT, - Language.SINDHI, - Language.TAMIL, - Language.TELUGU, - Language.URDU, - # Optionally: Maithili, Santali, Sindhi, Konkani - ] - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] -] diff --git a/src/lighteval/tasks/multilingual/tasks/enem.py b/src/lighteval/tasks/multilingual/tasks/enem.py deleted file mode 100644 index 478135dab..000000000 --- a/src/lighteval/tasks/multilingual/tasks/enem.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -name: -Enem - -dataset: -maritaca-ai/enem - -abstract: -ENEM (Exame Nacional do Ensino Médio) is a standardized Brazilian national -secondary education examination. The exam is used both as a university admission -test and as a high school evaluation test. - -languages: -portuguese - -tags: -knowledge, multilingual, multiple-choice - -paper: -https://huggingface.co/datasets/maritaca-ai/enem -""" - -from functools import partial - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.adapters import ( - enem_adapter, -) -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"enem_{Language.PORTUGUESE.value}_{formulation.name.lower()}:{year}", - prompt_function=get_mcq_prompt_function( - Language.PORTUGUESE, - partial( - enem_adapter, - Language.PORTUGUESE, - ), - formulation=formulation, - ), - hf_repo="maritaca-ai/enem", - hf_subset=year, - evaluation_splits=("train",), - hf_avail_splits=["train"], - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for year in ["2022", "2023", "2024"] - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/exams.py b/src/lighteval/tasks/multilingual/tasks/exams.py deleted file mode 100644 index 28e40e989..000000000 --- a/src/lighteval/tasks/multilingual/tasks/exams.py +++ /dev/null @@ -1,193 +0,0 @@ -""" -name: -Exams - -dataset: -mhardalov/exams - -abstract: -Exams multilingual benchmark. - -languages: -albanian, arabic, bulgarian, croatian, french, german, hungarian, italian, -lithuanian, macedonian, polish, portuguese, serbian, spanish, turkish, -vietnamese - -tags: -knowledge, multilingual, multiple-choice - -paper: -""" - -from functools import partial - -from langcodes import Language as LangCodeLanguage -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation, normalize_subset -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -exams_subjects_by_lang: dict[Language, set[str]] = { - Language.ARABIC: {"Biology", "Islamic Studies", "Physics", "Science", "Social"}, - Language.BULGARIAN: {"Biology", "Chemistry", "Geography", "History", "Philosophy", "Physics"}, - Language.CROATIAN: { - "Biology", - "Chemistry", - "Ethics", - "Fine Arts", - "Geography", - "Geology", - "History", - "Informatics", - "Philosophy", - "Physics", - "Politics", - "Psychology", - "Religion", - "Sociology", - }, - Language.HUNGARIAN: { - "Agriculture", - "Agriculture (Mechanical knowledge)", - "Biology", - "Chemistry", - "Economics", - "Economics & Marketing", - "Economics Basics (Business)", - "Economics Basics (Theoretical)", - "Forestry", - "Geography", - "Landscaping", - "Physics", - "Politics", - "Tourism", - }, - Language.ITALIAN: { - "Biology", - "Chemistry", - "Ethics", - "Geography", - "Geology", - "History", - "Informatics", - "Philosophy", - "Physics", - "Politics", - "Psychology", - "Sociology", - }, - Language.SERBIAN: { - "Biology", - "Chemistry", - "Ethics", - "Geography", - "Geology", - "History", - "Informatics", - "Philosophy", - "Physics", - "Politics", - "Psychology", - "Religion", - "Sociology", - }, - Language.FRENCH: {"Economics", "Economics & Marketing", "Economics Basics (Theoretical)", "Geography", "Physics"}, - Language.GERMAN: { - "Chemistry", - "Economics", - "Economics & Marketing", - "Economics Basics (Theoretical)", - "Geography", - "Physics", - "Tourism", - }, - Language.SPANISH: {"Geography", "Physics"}, - Language.LITHUANIAN: {"Geology", "History"}, - Language.ALBANIAN: { - "Biology", - "Business", - "Chemistry", - "Fine Arts", - "History", - "Philosophy", - "Physics", - "Sociology", - }, - Language.MACEDONIAN: { - "Biology", - "Business", - "Chemistry", - "Fine Arts", - "History", - "Philosophy", - "Physics", - "Sociology", - }, - Language.TURKISH: { - "Biology", - "Business", - "Chemistry", - "Geography", - "History", - "Philosophy", - "Physics", - "Sociology", - }, - Language.POLISH: {"Professional"}, - Language.PORTUGUESE: {"Biology", "Economics", "Geology", "Philosophy"}, - Language.VIETNAMESE: {"Biology", "Chemistry", "Citizenship", "Geography", "History", "Physics"}, -} - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"exams_{language.value}_{formulation.name.lower()}:{normalize_subset(subject)}", - prompt_function=get_mcq_prompt_function( - language, - lambda line: { - "question": line["question"]["stem"], - "choices": line["question"]["choices"]["text"], - "gold_idx": line["question"]["choices"]["label"].index(line["answerKey"]), - }, - formulation=formulation, - ), - hf_repo="mhardalov/exams", - hf_subset="multilingual", - # Weird bug in dataset - hf_filter=partial( - lambda language, subject, line: line["answerKey"] != "@" - and line["info"]["language"] == LangCodeLanguage(standardize_tag(language.value)).language_name() - and line["info"]["subject"] == subject, - language, - subject, - ), - evaluation_splits=("test",), - few_shots_split="train", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for language in exams_subjects_by_lang.keys() - for subject in exams_subjects_by_lang[language] - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/faquad.py b/src/lighteval/tasks/multilingual/tasks/faquad.py deleted file mode 100644 index 63ee46a57..000000000 --- a/src/lighteval/tasks/multilingual/tasks/faquad.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -name: -Faquad - -dataset: -eraldoluis/faquad - -abstract: -FaQuAD: A Portuguese Reading Comprehension Dataset - -languages: -portuguese - -tags: -multilingual, qa - -paper: -https://arxiv.org/abs/2007.15671 -""" - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"faquad_{Language.PORTUGUESE.value}", - prompt_function=get_qa_prompt_function( - Language.PORTUGUESE, - lambda line: { - "question": line["question"], - "context": line["context"], - "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0], - }, - ), - hf_repo="eraldoluis/faquad", - hf_subset="plain_text", - hf_revision="205ba826a2282a4a5aa9bd3651e55ee4f2da1546", - hf_filter=lambda line: any(len(ans) > 0 for ans in line["answers"]["text"]), - evaluation_splits=("validation",), - few_shots_split="train", - metrics=( - MultilingualQuasiExactMatchMetric(Language.PORTUGUESE, "prefix"), - MultilingualQuasiF1ScoreMetric(Language.PORTUGUESE), - ), - generation_size=400, - stop_sequence=("\n",), - ) -] diff --git a/src/lighteval/tasks/multilingual/tasks/filipino.py b/src/lighteval/tasks/multilingual/tasks/filipino.py deleted file mode 100644 index 5138c49eb..000000000 --- a/src/lighteval/tasks/multilingual/tasks/filipino.py +++ /dev/null @@ -1,817 +0,0 @@ -""" -name: -Filipino Evals - -dataset: -filbench/filbench-eval - -abstract: -Collection of benchmarks for Filipino language. - -languages: -filipino - -tags: -knowledge, multilingual, multiple-choice - -paper: -https://github.com/filbench/filbench-eval/blob/main/filbench.pdf - -Contact: -- Lester James V. Miranda -- Elyanah Aco -- Conner Manuel -- Jan Christian Blaise Cruz -- Joseph Imperial -""" - -from collections import OrderedDict -from functools import partial -from string import ascii_uppercase -from typing import Any - -from langcodes import Language as LangCodeLanguage -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import LogLikelihoodAccMetric -from lighteval.metrics.metrics import Metrics -from lighteval.metrics.normalizations import ( - LogProbCharNorm, - LogProbPMINorm, - LogProbTokenNorm, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.requests import Doc -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.nli import get_nli_prompt_function -from lighteval.tasks.templates.translation import get_translation_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language, iso_639_3_ind_to_iso_639_3_macro - - -MMLU_SUBSETS = [ - "abstract_algebra", - "anatomy", - "astronomy", - "business_ethics", - "clinical_knowledge", - "college_biology", - "college_chemistry", - "college_computer_science", - "college_mathematics", - "college_medicine", - "college_physics", - "computer_security", - "conceptual_physics", - "econometrics", - "electrical_engineering", - "elementary_mathematics", - "formal_logic", - "global_facts", - "high_school_biology", - "high_school_chemistry", - "high_school_computer_science", - "high_school_european_history", - "high_school_geography", - "high_school_government_and_politics", - "high_school_macroeconomics", - "high_school_mathematics", - "high_school_microeconomics", - "high_school_physics", - "high_school_psychology", - "high_school_statistics", - "high_school_us_history", - "high_school_world_history", - "human_aging", - "human_sexuality", - "international_law", - "jurisprudence", - "logical_fallacies", - "machine_learning", - "management", - "marketing", - "medical_genetics", - "miscellaneous", - "moral_disputes", - "moral_scenarios", - "nutrition", - "philosophy", - "prehistory", - "professional_accounting", - "professional_law", - "professional_medicine", - "professional_psychology", - "public_relations", - "security_studies", - "sociology", - "us_foreign_policy", - "virology", - "world_religions", -] - -# Balita NLP -FILIPINO_BALITA_TASKS = [ - LightevalTaskConfig( - name=f"balita_tgl_{formulation.name.lower()}", - prompt_function=get_mcq_prompt_function( - language=Language.TAGALOG, - adapter=lambda line: { - "question": "Alin sa mga titlulong nakalista sa ibaba ang pinaka-angkop para sa teksto?", - "context": f"Teksto: {line['title_choice_first_paragraph']}", - "choices": line["title_choices"], - "gold_idx": line["title_choice_gold_idx"], - }, - formulation=formulation, - ), - hf_repo="LanceBunag/BalitaNLP", - hf_subset="no-image", - hf_avail_splits=["train", "validation", "test"], - evaluation_splits=("validation", "test"), - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [MCFFormulation(), HybridFormulation()] -] - -# Belebele -FILIPINO_BELEBELE_TASKS = [ - LightevalTaskConfig( - name=f"belebele_{LangCodeLanguage.get(language).to_alpha3()}_{formulation.name.lower()}", - prompt_function=get_mcq_prompt_function( - iso_639_3_ind_to_iso_639_3_macro[LangCodeLanguage.get(language).to_alpha3()], - lambda line: { - "question": line["question"], - "context": line["flores_passage"], - "choices": [line[f"mc_answer{i}"] for i in range(1, 5)], - "gold_idx": int(line["correct_answer_num"]) - 1, - }, - formulation=formulation, - ), - hf_repo="facebook/belebele", - hf_subset=language, - evaluation_splits=("test",), - hf_avail_splits=["test"], - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] - for language in ["tgl_Latn", "ceb_Latn"] -] - -# CebuaNER -cebuaner_choices = ["PERSON", "ORGANIZATION", "LOCATION", "OTHER"] -cebuaner_answer_idx = ["A", "B", "C", "D"] -question = "Unsa ang ginganlan nga named-entity sa pulong '{entity}' niini nga sentence: {text}" -FILIPINO_CEBUANER_TASKS = [ - LightevalTaskConfig( - name=f"cebuaner_ceb_{formulation.name.lower()}", - hf_subset="default", - prompt_function=get_mcq_prompt_function( - Language.CEBUANO, - lambda line: { - "question": question.format(entity=line["entity"], text=line["text"]), - "choices": cebuaner_choices, - "gold_idx": cebuaner_answer_idx.index(line["answer"]), - }, - formulation=formulation, - ), - hf_repo="UD-Filipino/cebuaner-instruction", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split="test", - few_shots_select="random", - generation_size=-1, - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbPMINorm()), - ], - ), - version=0, - ) - for formulation in [MCFFormulation(), HybridFormulation()] -] - -# Cebuano Readability -cebuano_readability_choices = ["Grade 1", "Grade 2", "Grade 3"] -cebuano_readability_instruction = """ -Unsa ang angay nga lebel sa grado alang sa mosunod nga teksto? - -Grade 1 - ang teksto mahimong basahon sa usa ka tawo tali sa edad nga 6-7. -Grade 2 - ang teksto mahimong basahon sa usa ka tawo tali sa edad nga 7-8. -Grade 3 - ang teksto mahimong basahon sa usa ka tawo tali sa edad nga 8-9. -""" -FILIPINO_READABILITY_TASKS = [ - LightevalTaskConfig( - name=f"readability_ceb_{formulation.name.lower()}", - prompt_function=get_mcq_prompt_function( - Language.CEBUANO, - lambda line: { - "question": cebuano_readability_instruction + line["text"], - "choices": cebuano_readability_choices, - "gold_idx": cebuano_readability_choices.index(f"Grade {line['label']}"), - }, - formulation=formulation, - ), - hf_subset="default", - hf_repo="UD-Filipino/cebuano-readability", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbPMINorm()), - ], - ), - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split="test", - few_shots_select="random", - generation_size=-1, - version=0, - ) - for formulation in [MCFFormulation(), HybridFormulation()] -] - -# Dengue -dengue_filipino_subsets = { - "absent": "pagiging absent", - "dengue": "dengue", - "health": "kalusugan", - "mosquito": "lamok", - "sick": "sakit", -} - - -def filipino_dengue_pfn(line, task_name: str) -> Doc: - subset = task_name.split(":")[-1] - subset_keyword = dengue_filipino_subsets[subset] - - instruction = f"Tungkol ba sa {subset_keyword} ang sumusunod na pangungusap? Piliin ang tamang sagot:\n\n" - choices: dict[str, str] = OrderedDict({"A": "Hindi", "B": "Oo"}) - - answer_index = int(line.get(subset)) - query = f"{instruction}{line['text']}\n" - query += "".join([f"{key}. {choice}\n" for key, choice in choices.items()]) - query += "Sagot:" - return Doc( - task_name=task_name, - query=query, - choices=list(choices.keys()), - gold_index=answer_index, - instruction=instruction, - ) - - -FILIPINO_DENGUE_TASKS = [ - LightevalTaskConfig( - name=f"dengue_filipino_fil:{subset}", - hf_subset="default", - prompt_function=filipino_dengue_pfn, - hf_repo="jcblaise/dengue_filipino", - metrics=[LogLikelihoodAccMetric(normalization=LogProbTokenNorm())], - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["train"], - few_shots_split="train", - few_shots_select="random", - generation_size=-1, - version=0, - ) - for subset in dengue_filipino_subsets -] - -# FireCS -firecs_choices = ["Negatibo", "Neutral", "Positibo"] - -FILIPINO_FIRECS_TASK = [ - LightevalTaskConfig( - name=f"firecs_fil_{formulation.name.lower()}", - hf_subset="default", - prompt_function=get_mcq_prompt_function( - Language.TAGALOG, - lambda line: { - "question": f"Ano ang damdamin o sentimiyento ng sumusunod na pangungusap: {line['review']}", - "choices": firecs_choices, - "gold_idx": int(line["label"]), - }, - ), - hf_repo="ccosme/FiReCS", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbPMINorm()), - ], - ), - hf_avail_splits=["train", "test"], - evaluation_splits=["train"], - few_shots_split="train", - few_shots_select="random", - generation_size=-1, - version=0, - ) - for formulation in [MCFFormulation(), HybridFormulation()] -] - -# Global-MMLU (FIl) - -FILIPINO_GLOBAL_MMLU_TASKS = [ - LightevalTaskConfig( - name=f"global_mmlu_{sensitivity_label.lower()}_{language.value}_{formulation.name.lower()}:{subset}", - prompt_function=get_mcq_prompt_function( - language, - lambda line: { - "question": line["question"], - "choices": [ - line["option_a"], - line["option_b"], - line["option_c"], - line["option_d"], - ], - "gold_idx": ascii_uppercase.index(line["answer"]), - }, - formulation=formulation, - ), - hf_repo="CohereForAI/Global-MMLU", - hf_subset=standardize_tag(language.value), - evaluation_splits=("test",), - few_shots_split="dev", - hf_filter=partial( - lambda subset, sensitivity_label, x: x["subject"].lower() == subset - and ( - sensitivity_label == "ALL" or sensitivity_label in x["cultural_sensitivity_label"].replace("-", "UNK") - ), - subset, - sensitivity_label, - ), - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbPMINorm()), - ], - ), - ) - for subset in MMLU_SUBSETS - for language in [Language.TAGALOG] - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] - for sensitivity_label in ["ALL", "CA", "CS", "UNK"] -] - -# INCLUDE - -FILIPINO_INCLUDE_TASKS = [ - LightevalTaskConfig( - name=f"include_{language.value}_{formulation.name.lower()}:{subset}", - prompt_function=get_mcq_prompt_function( - language, - lambda line: { - "question": line["question"], - "choices": [line[f"option_{i}"] for i in ("a", "b", "c", "d")], - "gold_idx": line["answer"], - }, - formulation=formulation, - ), - hf_subset="Tagalog", - hf_repo="CohereForAI/include-base-44", - hf_filter=partial(lambda subset, x: x["subject"].replace(" ", "_").lower() == subset, subset), - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbPMINorm()), - ], - ), - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split="test", - few_shots_select="random", - generation_size=-1, - version=0, - ) - for subset in ["culturology", "history", "language", "driving_license"] - for language in [Language.TAGALOG] - for formulation in [MCFFormulation(), HybridFormulation()] -] - -# KALAHI -FILIPINO_KALAHI_TASKS = [ - LightevalTaskConfig( - name=f"kalahi_tgl_{formulation.name.lower()}", - prompt_function=get_mcq_prompt_function( - language=Language.TAGALOG, - adapter=lambda line: { - "question": line["prompts"][0]["question"], - "choices": [entry[3:] for entry in line["prompts"][0]["mcq"].split("\n")], - "gold_idx": ascii_uppercase.index(line["label"]), - }, - formulation=formulation, - ), - hf_repo="aisingapore/cultural_evaluation-kalahi", - hf_subset="default", - evaluation_splits=["tl"], - metrics=[ - LogLikelihoodAccMetric(normalization=None), - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ) - for formulation in [HybridFormulation(), MCFFormulation()] -] - -# NewsPH NLI -FILIPINO_NEWSPH_NLI_TASKS = [ - LightevalTaskConfig( - name=f"newsphnli_fil_{formulation.name.lower()}", - prompt_function=get_nli_prompt_function( - language=Language.TAGALOG, - adapter=lambda line: { - "premise": line["premise"], - "hypothesis": line["hypothesis"], - # Since there is no neutral label - "gold_idx": line["label"], - }, - relations=["entailment", "contradiction"], - formulation=formulation, - ), - hf_repo="jcblaise/newsph_nli", - hf_subset="default", - evaluation_splits=["validation"], - few_shots_split="train", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=None), - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] -] - -# NTREX-128 -FILIPINO_NTREX_TASK = [ - LightevalTaskConfig( - name=f"ntrex128_{LangCodeLanguage.get(language).to_alpha3()}", - prompt_function=get_translation_prompt_function( - source_language=Language.ENGLISH, - target_language=iso_639_3_ind_to_iso_639_3_macro[LangCodeLanguage.get(language).to_alpha3()], - adapter=lambda line: { - "source_text": line["eng_Latn"], - "target_text": line[language], - }, - formulation=CFFormulation(), - ), - hf_repo="mteb/NTREX", - hf_subset="default", - metrics=[ - Metrics.rougeL, - Metrics.bleu, - Metrics.bleurt, - Metrics.chrf, - Metrics.ter, - ], - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=64, - version=0, - ) - for language in ["fil_Latn"] -] - -# SIB-200 - -sib200_choices = [ - "geography", - "science/technology", - "entertainment", - "travel", - "sports", - "health", - "politics", -] - - -def get_instruction(language: Language) -> str: - if language == Language.CEBUANO: - return "Mahitungod sa unsa ang mosunod nga teksto?\n" - if language == Language.TAGALOG: - return "Tungkol saan ang sumusunod na pangungusap?\n" - - -def create_sib200_task(language: Language, formulation): - return LightevalTaskConfig( - name=f"sib200_{language.value}_{formulation.name.lower()}", - prompt_function=get_mcq_prompt_function( - language, - lambda line: { - "question": get_instruction(language) + line["text"], - "choices": sib200_choices, - "gold_idx": sib200_choices.index(line["category"]), - }, - formulation=formulation, - ), - hf_subset=f"{language.value}_Latn", - hf_repo="Davlan/sib200", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbPMINorm()), - ], - ), - hf_avail_splits=["test", "validation"], - evaluation_splits=["validation"], - few_shots_split="validation", - few_shots_select="random", - generation_size=-1, - version=0, - ) - - -FILIPINO_SIB_TASKS = [ - create_sib200_task(language, formulation) - for language in [Language.TAGALOG, Language.CEBUANO] - for formulation in [MCFFormulation(), HybridFormulation()] -] - - -def prepare_stingray_correctness(line: dict[str, str]) -> dict[str, Any]: - # lang2 is Tagalog - word = line["word"] - sentence = line["lang2_sentence"] - question = f"Is the usage of {word} in this sentence correct? \n{sentence}" - choices = ["Yes", "No"] - gold_idx = choices.index(line["usage_correctness_lang2_answer"]) - return {"question": question, "choices": choices, "gold_idx": gold_idx} - - -def prepare_stingray_semantic_appropriateness(line: dict[str, str]) -> dict[str, Any]: - lang1 = line["lang1_sentence"] - lang2 = line["lang2_sentence"] - question = "Which sentence is more semantically appropriate?" - choices = [lang1, lang2, "Both"] - choice_letters = ["A", "B", "C"] - gold_idx = choice_letters.index(line["semantic_appropriate_answer"]) - return {"question": question, "choices": choices, "gold_idx": gold_idx} - - -FILIPINO_STINGRAY_CORRECTNESS_TASKS = [ - LightevalTaskConfig( - name=f"stingraybench_correctness_tgl_{formulation.name.lower()}", - prompt_function=get_mcq_prompt_function( - Language.ENGLISH, # the orig instruction is in English, so we replicate it. - adapter=prepare_stingray_correctness, - formulation=formulation, - ), - hf_subset="id_tl", - hf_repo="StingrayBench/StingrayBench", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbPMINorm()), - ], - ), - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split="test", - few_shots_select="random", - generation_size=-1, - version=0, - ) - for formulation in [MCFFormulation(), HybridFormulation()] -] - -FILIPINO_STINGRAY_SEMANTIC_TASKS = [ - LightevalTaskConfig( - name=f"stingraybench_semantic_appropriateness_tgl_{formulation.name.lower()}", - prompt_function=get_mcq_prompt_function( - Language.ENGLISH, # the orig instruction is in English, so we replicate it. - adapter=prepare_stingray_semantic_appropriateness, - formulation=formulation, - ), - hf_subset="id_tl", - hf_repo="StingrayBench/StingrayBench", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbPMINorm()), - ], - ), - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split="test", - few_shots_select="random", - generation_size=-1, - version=0, - ) - for formulation in [MCFFormulation(), HybridFormulation()] -] - -FILIPINO_STINGRAY_TASKS = FILIPINO_STINGRAY_SEMANTIC_TASKS + FILIPINO_STINGRAY_CORRECTNESS_TASKS - -# Tatoeba -# We follow the original translation direction from tatoeba -lang_dict = { - "ceb": { - "subset": "ceb-eng", - "source_language": Language.CEBUANO, - "target_language": Language.ENGLISH, - }, - "tgl": { - "subset": "eng-tgl", - "source_language": Language.ENGLISH, - "target_language": Language.TAGALOG, - }, -} - -FILIPINO_TATOEBA_TASKS = [ - LightevalTaskConfig( - name=f"tatoeba_{language}", - prompt_function=get_translation_prompt_function( - source_language=meta.get("source_language"), - target_language=meta.get("target_language"), - adapter=lambda line: { - "source_text": line["sourceString"], - "target_text": line["targetString"], - }, - formulation=CFFormulation(), - ), - hf_repo="Helsinki-NLP/tatoeba_mt", - hf_subset=meta.get("subset"), - metrics=[ - Metrics.rougeL, - Metrics.bleu, - Metrics.bleurt, - Metrics.chrf, - Metrics.ter, - ], - hf_avail_splits=["test"], - evaluation_splits=["test"], - generation_size=64, - ) - for language, meta in lang_dict.items() -] - -# TICO-19 -FILIPINO_TICO19_TASKS = [ - LightevalTaskConfig( - name="tico19_tgl", - prompt_function=get_translation_prompt_function( - source_language=Language.ENGLISH, - target_language=Language.TAGALOG, - adapter=lambda line: { - "source_text": line["sourceString"], - "target_text": line["targetString"], - }, - formulation=CFFormulation(), - ), - hf_repo="gmnlp/tico19", - hf_subset="en-tl", - metrics=[ - Metrics.rougeL, - Metrics.bleu, - Metrics.bleurt, - Metrics.chrf, - Metrics.ter, - ], - hf_avail_splits=["test", "validation"], - evaluation_splits=["validation"], - few_shots_split=["validation"], - few_shots_select="random", - generation_size=64, - ) -] - -# TLUnified-NER -tlunified_ner_choices = ["PERSON", "ORGANIZATION", "LOCATION"] -tlunified_ner_answer_idx = ["A", "B", "C"] - -FILIPINO_TLUNIFIED_NER_TASK = [ - LightevalTaskConfig( - name=f"tlunifiedner_tgl_{formulation.name.lower()}", - hf_subset="instruction", - prompt_function=get_mcq_prompt_function( - Language.TAGALOG, - lambda line: { - "question": f"Ano ang named-entity ng salitang '{line['entity']}' sa pangungusap na ito: {line['text']}", - "choices": tlunified_ner_choices, - "gold_idx": tlunified_ner_answer_idx.index(line["answer"]), - }, - formulation=formulation, - ), - hf_repo="ljvmiranda921/tlunified-ner", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split="test", - few_shots_select="random", - generation_size=-1, - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbPMINorm()), - ], - ), - version=0, - ) - for formulation in [MCFFormulation(), HybridFormulation()] -] - -# Universal NER -universalner_choices = ["PERSON", "ORGANIZATION", "LOCATION"] -universalner_answer_idx = ["A", "B", "C"] - - -def create_universalner_task(language: Language, formulation): - if language == Language.CEBUANO: - question = "Unsa ang ginganlan nga named-entity sa pulong '{entity}' niini nga sentence: {text}" - if language == Language.TAGALOG: - question = "Ano ang named-entity ng salitang '{entity}' sa pangungusap na ito: {text}" - - return LightevalTaskConfig( - name=f"universalner_{language.value}_{formulation.name.lower()}", - hf_subset=language.value, - prompt_function=get_mcq_prompt_function( - language, - lambda line: { - "question": question.format(entity=line["entity"], text=line["text"]), - "choices": universalner_choices, - "gold_idx": universalner_answer_idx.index(line["answer"]), - }, - formulation=formulation, - ), - hf_repo="UD-Filipino/universalner-instruction", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split="test", - few_shots_select="random", - generation_size=-1, - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbPMINorm()), - ], - ), - version=0, - ) - - -FILIPINO_UNIVERSALNER_TASKS = [ - create_universalner_task(language, formulation) - for language in [Language.CEBUANO, Language.TAGALOG] - for formulation in [MCFFormulation(), HybridFormulation()] -] - -# Tasks Table - -TASKS_TABLE: list[LightevalTaskConfig] = ( - FILIPINO_BALITA_TASKS - + FILIPINO_BELEBELE_TASKS - + FILIPINO_CEBUANER_TASKS - + FILIPINO_READABILITY_TASKS - + FILIPINO_DENGUE_TASKS - + FILIPINO_FIRECS_TASK - + FILIPINO_GLOBAL_MMLU_TASKS - + FILIPINO_INCLUDE_TASKS - + FILIPINO_KALAHI_TASKS - + FILIPINO_NEWSPH_NLI_TASKS - + FILIPINO_NTREX_TASK - + FILIPINO_SIB_TASKS - + FILIPINO_STINGRAY_TASKS - + FILIPINO_TATOEBA_TASKS - + FILIPINO_TICO19_TASKS - + FILIPINO_TLUNIFIED_NER_TASK - + FILIPINO_UNIVERSALNER_TASKS -) diff --git a/src/lighteval/tasks/multilingual/tasks/flores200.py b/src/lighteval/tasks/multilingual/tasks/flores200.py deleted file mode 100644 index f6adbef78..000000000 --- a/src/lighteval/tasks/multilingual/tasks/flores200.py +++ /dev/null @@ -1,270 +0,0 @@ -""" -name: -Flores200 - -dataset: -facebook/flores - -abstract: -Flores200 multilingual benchmark. - -languages: -arabic, armenian, bengali, cyrillic, devanagari, ethiopic, georgian, greek, -gujarati, gurmukhi, chinese (simplified), chinese (traditional), hangul, hebrew, -japanese, khmer, kannada, lao, latin, malayalam, myanmar, odia, sinhala, tamil, -telugu, thai, tibetan - -tags: -multilingual, translation - -paper: -""" - -from itertools import permutations - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.translation import get_translation_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, -) -from lighteval.utils.language import Language, manage_duplicate_language_codes - - -flores_200_languages = [ - # "ace_Arab", - "ace_Latn", - "acm_Arab", - "acq_Arab", - "aeb_Arab", - "afr_Latn", - "ajp_Arab", - "aka_Latn", - "amh_Ethi", - "apc_Arab", - "arb_Arab", - # "arb_Latn", - "ars_Arab", - "ary_Arab", - "arz_Arab", - "asm_Beng", - "ast_Latn", - "awa_Deva", - "ayr_Latn", - "azb_Arab", - "azj_Latn", - "bak_Cyrl", - "bam_Latn", - "ban_Latn", - "bel_Cyrl", - "bem_Latn", - "ben_Beng", - "bho_Deva", - # "bjn_Arab", - "bjn_Latn", - "bod_Tibt", - "bos_Latn", - "bug_Latn", - "bul_Cyrl", - "cat_Latn", - "ceb_Latn", - "ces_Latn", - "cjk_Latn", - "ckb_Arab", - "crh_Latn", - "cym_Latn", - "dan_Latn", - "deu_Latn", - "dik_Latn", - "dyu_Latn", - "dzo_Tibt", - "ell_Grek", - "eng_Latn", - "epo_Latn", - "est_Latn", - "eus_Latn", - "ewe_Latn", - "fao_Latn", - "fij_Latn", - "fin_Latn", - "fon_Latn", - "fra_Latn", - "fur_Latn", - "fuv_Latn", - "gla_Latn", - "gle_Latn", - "glg_Latn", - "grn_Latn", - "guj_Gujr", - "hat_Latn", - "hau_Latn", - "heb_Hebr", - "hin_Deva", - "hne_Deva", - "hrv_Latn", - "hun_Latn", - "hye_Armn", - "ibo_Latn", - "ilo_Latn", - "ind_Latn", - "isl_Latn", - "ita_Latn", - "jav_Latn", - "jpn_Jpan", - "kab_Latn", - "kac_Latn", - "kam_Latn", - "kan_Knda", - # "kas_Arab", - "kas_Deva", - "kat_Geor", - # "knc_Arab", - "knc_Latn", - "kaz_Cyrl", - "kbp_Latn", - "kea_Latn", - "khm_Khmr", - "kik_Latn", - "kin_Latn", - "kir_Cyrl", - "kmb_Latn", - "kmr_Latn", - "kon_Latn", - "kor_Hang", - "lao_Laoo", - "lij_Latn", - "lim_Latn", - "lin_Latn", - "lit_Latn", - "lmo_Latn", - "ltg_Latn", - "ltz_Latn", - "lua_Latn", - "lug_Latn", - "luo_Latn", - "lus_Latn", - "lvs_Latn", - "mag_Deva", - "mai_Deva", - "mal_Mlym", - "mar_Deva", - # "min_Arab", - "min_Latn", - "mkd_Cyrl", - "plt_Latn", - "mlt_Latn", - "mni_Beng", - "khk_Cyrl", - "mos_Latn", - "mri_Latn", - "mya_Mymr", - "nld_Latn", - "nno_Latn", - "nob_Latn", - "npi_Deva", - "nso_Latn", - "nus_Latn", - "nya_Latn", - "oci_Latn", - "gaz_Latn", - "ory_Orya", - "pag_Latn", - "pan_Guru", - "pap_Latn", - "pes_Arab", - "pol_Latn", - "por_Latn", - "prs_Arab", - "pbt_Arab", - "quy_Latn", - "ron_Latn", - "run_Latn", - "rus_Cyrl", - "sag_Latn", - "san_Deva", - "sat_Olck", - "scn_Latn", - "shn_Mymr", - "sin_Sinh", - "slk_Latn", - "slv_Latn", - "smo_Latn", - "sna_Latn", - "snd_Arab", - "som_Latn", - "sot_Latn", - "spa_Latn", - "als_Latn", - "srd_Latn", - "srp_Cyrl", - "ssw_Latn", - "sun_Latn", - "swe_Latn", - "swh_Latn", - "szl_Latn", - "tam_Taml", - "tat_Cyrl", - "tel_Telu", - "tgk_Cyrl", - "tgl_Latn", - "tha_Thai", - "tir_Ethi", - "taq_Latn", - "taq_Tfng", - "tpi_Latn", - "tsn_Latn", - "tso_Latn", - "tuk_Latn", - "tum_Latn", - "tur_Latn", - "twi_Latn", - "tzm_Tfng", - "uig_Arab", - "ukr_Cyrl", - "umb_Latn", - "urd_Arab", - "uzn_Latn", - "vec_Latn", - "vie_Latn", - "war_Latn", - "wol_Latn", - "xho_Latn", - "ydd_Hebr", - "yor_Latn", - "yue_Hant", - "zho_Hans", - # "zho_Hant", - "zsm_Latn", - "zul_Latn", -] - - -def flores_adapter(lang1, lang2): - return lambda line: { - "source_text": line[f"sentence_{lang1}"], - "target_text": line[f"sentence_{lang2}"], - } - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"flores200:{lang1}-{lang2}", - prompt_function=get_translation_prompt_function( - source_language=Language(manage_duplicate_language_codes(lang1.split("_")[0])), - target_language=Language(manage_duplicate_language_codes(lang2.split("_")[0])), - adapter=flores_adapter(lang1, lang2), - formulation=CFFormulation(), - ), - hf_repo="facebook/flores", - hf_subset=f"{lang1}-{lang2}", - hf_avail_splits=["dev", "devtest"], - evaluation_splits=["devtest"], - few_shots_split="dev", - few_shots_select=None, - generation_size=300, - metrics=[Metrics.chrf_plus, Metrics.bleu, Metrics.bleu_1, Metrics.bleu_4], - stop_sequence=["\n"], - version=0, - ) - for (lang1, lang2) in permutations(flores_200_languages, 2) -] diff --git a/src/lighteval/tasks/multilingual/tasks/fquad_v2.py b/src/lighteval/tasks/multilingual/tasks/fquad_v2.py deleted file mode 100644 index 777df03b8..000000000 --- a/src/lighteval/tasks/multilingual/tasks/fquad_v2.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -name: -Fquad V2 - -dataset: -manu/fquad2_test - -abstract: -FQuAD v2: French Question Answering Dataset version 2. - -languages: -french - -tags: -multilingual, qa - -paper: -https://arxiv.org/abs/2002.06071 -""" - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"fquadv2_{Language.FRENCH.value}", - prompt_function=get_qa_prompt_function( - Language.FRENCH, - lambda line: { - "question": line["question"], - "context": line["context"], - "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0], - }, - ), - hf_repo="manu/fquad2_test", - hf_subset="default", - evaluation_splits=("test_hasAns",), - few_shots_split="valid_hasAns", - generation_size=400, - stop_sequence=("\n",), - metrics=( - MultilingualQuasiExactMatchMetric(Language.FRENCH, "prefix"), - MultilingualQuasiF1ScoreMetric(Language.FRENCH), - ), - ) -] diff --git a/src/lighteval/tasks/multilingual/tasks/french.py b/src/lighteval/tasks/multilingual/tasks/french.py deleted file mode 100644 index 509bf5740..000000000 --- a/src/lighteval/tasks/multilingual/tasks/french.py +++ /dev/null @@ -1,132 +0,0 @@ -""" -name: -French Evals - -dataset: -fr-gouv-coordination-ia/IFEval-fr, fr-gouv-coordination-ia/gpqa-fr, fr-gouv-coordination-ia/bac-fr - -abstract: -Collection of benchmarks for the french language. - -languages: -french - -tags: -knowledge, multiple-choice, qa - -paper: -https://huggingface.co/fr-gouv-coordination-ia -""" - -import random -from string import ascii_uppercase - -from lighteval.metrics.metrics import Metrics -from lighteval.metrics.normalizations import math_normalizer -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc -from lighteval.tasks.tasks.ifeval.main import ifeval_metrics -from lighteval.utils.utils import as_list - - -# Ifeval-fr prompt function -def prompt_ifeval_fr(line, task_name: str = None): - return Doc( - task_name=task_name, - query=line["prompt"], - choices=[""], - gold_index=0, - instruction="", - specific={"instructions_id_list": line["instruction_id_list"], "kwargs": line["kwargs"]}, - ) - - -# qpqa-fr prompt function -def prompt_gpqa_fr(line, task_name: str = None): - gold_index = random.randint(0, 3) - choices = [line["Réponse incorrecte 1"], line["Réponse incorrecte 2"], line["Réponse incorrecte 3"]] - choices.insert(gold_index, line["Réponse correcte"]) - - instruction = "Choisissez la réponse correcte aux questions suivantes.\n\n" - - query = f"Question: {line['Question']}\n" - query += "".join([f"{key}. {choice}\n" for key, choice in zip(ascii_uppercase, choices)]) - query += "Réponse: " - return Doc( - task_name=task_name, - query=f"{instruction}{query}", - choices=ascii_uppercase[: len(choices)], - gold_index=gold_index, - instruction=instruction, - ) - - -# BAC-fr prompt function -def prompt_bac_fr(line, task_name: str = None): - prompt = f"Enoncé: {line['enonce']}\n{line['instruction']}\n" - if line["choix"] is not None: # Multichoice evaluation - # prompt += "\n".join([f"{ascii_uppercase[ix]}.{choix}" for ix, choix in enumerate(line["choix"])]) - return Doc( - task_name=task_name, - query=prompt, - choices=as_list(line["choix"]), - gold_index=line["choix"].index(line["choix correct"]), - instruction="", - ) - else: - return Doc(task_name=task_name, query=prompt, choices=[line["reponse"]], gold_index=0, instruction="") - - -# IFEVal-fr task -ifeval_fr_task = LightevalTaskConfig( - name="ifeval-fr", - prompt_function=prompt_ifeval_fr, # must be defined in the file or imported from src/lighteval/tasks/tasks_prompt_formatting.py - hf_repo="fr-gouv-coordination-ia/IFEval-fr", - hf_subset="default", - metrics=[ifeval_metrics], - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split="train", - few_shots_select="random_sampling", - generation_size=1280, - stop_sequence=[], # no stop sequence, will use eot token - version="0.1", # select your metric in Metrics -) - -# GPQA-fr task -gpqa_fr_task = LightevalTaskConfig( - name="gpqa-fr", - prompt_function=prompt_gpqa_fr, - hf_repo="fr-gouv-coordination-ia/gpqa-fr", - hf_subset="default", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -# BAC-fr task -bac_fr_task = LightevalTaskConfig( - name="bac-fr", - prompt_function=prompt_bac_fr, - hf_repo="fr-gouv-coordination-ia/bac-fr", - hf_subset="default", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=1, - metrics=[ - Metrics.exact_match(sample_params={"normalize_gold": math_normalizer, "normalize_pred": math_normalizer}), - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -# STORE YOUR EVALS -TASKS_TABLE = [ifeval_fr_task, gpqa_fr_task, bac_fr_task] diff --git a/src/lighteval/tasks/multilingual/tasks/french_boolq.py b/src/lighteval/tasks/multilingual/tasks/french_boolq.py deleted file mode 100644 index d6f3fb9bb..000000000 --- a/src/lighteval/tasks/multilingual/tasks/french_boolq.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -name: -French Boolq - -dataset: -manu/french_boolq - -abstract: -French Boolq multilingual benchmark. - -languages: -french - -tags: -classification, multilingual, qa - -paper: -""" - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, - MultilingualQuasiExactMatchMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.boolq import get_boolq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"community_boolq_{Language.FRENCH.value}", - prompt_function=get_boolq_prompt_function( - Language.FRENCH, - lambda line: { - "question": line["question"], - "answer": line["label"] == 1, - "context": line["passage"], - }, - formulation=CFFormulation(), - ), - hf_repo="manu/french_boolq", - hf_subset="default", - evaluation_splits=("test",), - few_shots_split="valid", - generation_size=5, - stop_sequence=["\n"], - metrics=[MultilingualQuasiExactMatchMetric(Language.FRENCH, "full"), LogLikelihoodAccMetric()], - ) -] diff --git a/src/lighteval/tasks/multilingual/tasks/french_triviqa.py b/src/lighteval/tasks/multilingual/tasks/french_triviqa.py deleted file mode 100644 index 48a66f5df..000000000 --- a/src/lighteval/tasks/multilingual/tasks/french_triviqa.py +++ /dev/null @@ -1,50 +0,0 @@ -""" -name: -French Triviqa - -dataset: -manu/french-trivia - -abstract: -French Triviqa multilingual benchmark. - -languages: -french - -tags: -multilingual, qa - -paper: -""" - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"community_triviaqa_{Language.FRENCH.value}", - prompt_function=get_qa_prompt_function( - Language.FRENCH, - lambda line: { - "question": line["Question"], - "choices": [line["Answer"]], - }, - ), - hf_repo="manu/french-trivia", - hf_subset="default", - evaluation_splits=("train",), - hf_avail_splits=["train"], - generation_size=400, - stop_sequence=("\n",), - metrics=[ - MultilingualQuasiExactMatchMetric(Language.FRENCH, "prefix"), - MultilingualQuasiF1ScoreMetric(Language.FRENCH), - ], - ) -] diff --git a/src/lighteval/tasks/multilingual/tasks/german_rag.py b/src/lighteval/tasks/multilingual/tasks/german_rag.py deleted file mode 100644 index 1c9dbaf23..000000000 --- a/src/lighteval/tasks/multilingual/tasks/german_rag.py +++ /dev/null @@ -1,207 +0,0 @@ -""" -name: -German RAG Evals - -dataset: -deutsche-telekom/Ger-RAG-eval - -abstract: -Collection of benchmarks for the German language. - -languages: -german - -tags: -knowledge, reasoning, multiple-choice - -paper: -https://huggingface.co/datasets/deutsche-telekom/Ger-RAG-eval -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def prompt_fn_choose_question_by_context(line, task_name: str = None): - instruction = "Welche der folgenden Fragen (A oder B oder C oder D) lässt sich anhand des Kontext beantworten?\n\n" - query_template = """\ -Kontext: -{context} - -Fragen: -A: {choice_a} -B: {choice_b} -C: {choice_c} -D: {choice_d} - -Antwort:""" - query = instruction + query_template.format( - context=line["context"], - choice_a=line["choice_a"], - choice_b=line["choice_b"], - choice_c=line["choice_c"], - choice_d=line["choice_d"], - ) - choices = ["A", "B", "C", "D"] - return Doc( - task_name=task_name, - instruction=instruction, - query=query, - choices=choices, - gold_index=choices.index(line["target"]), - ) - - -def prompt_fn_choose_context_by_question(line, task_name: str = None): - instruction = ( - "Auf Basis welcher der folgenden Kontexte (A oder B oder C oder D) lässt sich die Frage beantworten?\n\n" - ) - query_template = """\ -Frage: {question} - -Kontexte: - -A: -{choice_a} - -B: -{choice_b} - -C: -{choice_c} - -D: -{choice_d} - -Antwort:""" - query = instruction + query_template.format( - question=line["question"], - choice_a=line["choice_a"], - choice_b=line["choice_b"], - choice_c=line["choice_c"], - choice_d=line["choice_d"], - ) - choices = ["A", "B", "C", "D"] - return Doc( - task_name=task_name, - instruction=instruction, - query=query, - choices=choices, - gold_index=choices.index(line["target"]), - ) - - -def prompt_fn_question_answer_match(line, task_name: str = None): - instruction = "Beantwortet die Antwort wirklich die Frage? Antworte mit J für ja oder N für nein.\n\n" - query_template = """\ -Die Frage: {question} - -Die Antwort: {answer} - -Auswahl (J/N):""" - query = instruction + query_template.format( - question=line["question"], - answer=line["answer"], - ) - choices = ["J", "N"] - return Doc( - task_name=task_name, - instruction=instruction, - query=query, - choices=choices, - gold_index=choices.index(line["target"]), - ) - - -def prompt_fn_context_question_match(line, task_name: str = None): - instruction = "Lässt sich die Frage mithilfe der Informationen aus dem Kontext beantworten? Antworte mit J für ja oder N für nein.\n\n" - query_template = """\ -Kontext: -{context} - -Die Frage: {question} - -Auswahl (J/N):""" - query = instruction + query_template.format( - question=line["question"], - context=line["context"], - ) - choices = ["J", "N"] - return Doc( - task_name=task_name, - instruction=instruction, - query=query, - choices=choices, - gold_index=choices.index(line["target"]), - ) - - -# Task 1: Choose question by context. -# Given is a context and 4 questions. -# The task is to decide which question can be answered by the context. -task1 = LightevalTaskConfig( - name="german_rag_eval:choose_question_by_context", - prompt_function=prompt_fn_choose_question_by_context, - hf_repo="deutsche-telekom/Ger-RAG-eval", - hf_subset="task1", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split="test", - few_shots_select="sequential", - metrics=[Metrics.loglikelihood_acc], - version=1, -) - -# Task 2: Choose context by question. -# Given is a question and 4 contexts. -# The task is to decide which context can answer the question. -task2 = LightevalTaskConfig( - name="german_rag_eval:choose_context_by_question", - prompt_function=prompt_fn_choose_context_by_question, - hf_repo="deutsche-telekom/Ger-RAG-eval", - hf_subset="task2", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split="test", - few_shots_select="sequential", - metrics=[Metrics.loglikelihood_acc], - version=1, -) - - -# Task 3: Question-answer match. -# Given is a question and an answer. -# The task is to decide whether the answer actually answers the question. -task3 = LightevalTaskConfig( - name="german_rag_eval:question_answer_match", - prompt_function=prompt_fn_question_answer_match, - hf_repo="deutsche-telekom/Ger-RAG-eval", - hf_subset="task3", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split="test", - few_shots_select="sequential", - metrics=[Metrics.loglikelihood_acc], - version=1, -) - -# Task 4: Context-question match. -# Given is a context and a question. -# The task is to decide whether the question can be answered by the context or not. -task4 = LightevalTaskConfig( - name="german_rag_eval:context_question_match", - prompt_function=prompt_fn_context_question_match, - hf_repo="deutsche-telekom/Ger-RAG-eval", - hf_subset="task4", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split="test", - few_shots_select="sequential", - metrics=[Metrics.loglikelihood_acc], - version=1, -) - - -# STORE YOUR EVALS -TASKS_TABLE = [task1, task2, task3, task4] diff --git a/src/lighteval/tasks/multilingual/tasks/germanquad.py b/src/lighteval/tasks/multilingual/tasks/germanquad.py deleted file mode 100644 index 69f1286d1..000000000 --- a/src/lighteval/tasks/multilingual/tasks/germanquad.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -name: -Germanquad - -dataset: -deepset/germanquad - -abstract: -GermanQuAD: High-quality German QA dataset with 13,722 questions. - -languages: -german - -tags: -multilingual, qa - -paper: -https://arxiv.org/abs/2104.12741 -""" - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"germanquad_{Language.GERMAN.value}", - prompt_function=get_qa_prompt_function( - Language.GERMAN, - lambda line: { - "question": line["question"], - "context": line["context"], - "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0], - }, - ), - hf_repo="deepset/germanquad", - hf_subset="plain_text", - hf_revision="fff05ceaf2ffbe5b65c7e0c57e678f7b7e1a0581", - hf_filter=lambda line: any(len(ans) > 0 for ans in line["answers"]["text"]), - evaluation_splits=("test",), - few_shots_split="train", - generation_size=400, - stop_sequence=("\n",), - metrics=( - MultilingualQuasiExactMatchMetric(Language.GERMAN, "prefix"), - MultilingualQuasiF1ScoreMetric(Language.GERMAN), - ), - ) -] diff --git a/src/lighteval/tasks/multilingual/tasks/global_mmlu.py b/src/lighteval/tasks/multilingual/tasks/global_mmlu.py deleted file mode 100644 index 894f15a3c..000000000 --- a/src/lighteval/tasks/multilingual/tasks/global_mmlu.py +++ /dev/null @@ -1,179 +0,0 @@ -""" -name: -Global Mmlu - -dataset: -CohereForAI/Global-MMLU - -abstract: -Translated MMLU using both professional and non-professional translators. -Contains tags for cultural sensitivity. - -languages: -amharic, arabic, bengali, chinese, czech, dutch, english, french, german, -hebrew, hindi, indonesian, italian, japanese, korean, malay, norwegian, polish, -portuguese, romanian, russian, serbian, spanish, swahili, swedish, tamil, -telugu, thai, turkish, ukrainian, urdu, vietnamese, yoruba, zulu - -tags: -knowledge, multilingual, multiple-choice - -paper: -https://huggingface.co/papers/2412.03304 -""" - -from functools import partial -from string import ascii_uppercase - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbPMINorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - MCFFormulation, -) -from lighteval.utils.language import Language - - -MMLU_SUBSETS = [ - "abstract_algebra", - "anatomy", - "astronomy", - "business_ethics", - "clinical_knowledge", - "college_biology", - "college_chemistry", - "college_computer_science", - "college_mathematics", - "college_medicine", - "college_physics", - "computer_security", - "conceptual_physics", - "econometrics", - "electrical_engineering", - "elementary_mathematics", - "formal_logic", - "global_facts", - "high_school_biology", - "high_school_chemistry", - "high_school_computer_science", - "high_school_european_history", - "high_school_geography", - "high_school_government_and_politics", - "high_school_macroeconomics", - "high_school_mathematics", - "high_school_microeconomics", - "high_school_physics", - "high_school_psychology", - "high_school_statistics", - "high_school_us_history", - "high_school_world_history", - "human_aging", - "human_sexuality", - "international_law", - "jurisprudence", - "logical_fallacies", - "machine_learning", - "management", - "marketing", - "medical_genetics", - "miscellaneous", - "moral_disputes", - "moral_scenarios", - "nutrition", - "philosophy", - "prehistory", - "professional_accounting", - "professional_law", - "professional_medicine", - "professional_psychology", - "public_relations", - "security_studies", - "sociology", - "us_foreign_policy", - "virology", - "world_religions", -] - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"global_mmlu_{sensitivity_label.lower()}_{language.value}_{formulation.name.lower()}:{subset}", - prompt_function=get_mcq_prompt_function( - language, - lambda line: { - "question": line["question"], - "choices": [line["option_a"], line["option_b"], line["option_c"], line["option_d"]], - "gold_idx": ascii_uppercase.index(line["answer"]), - }, - formulation=formulation, - ), - hf_repo="CohereForAI/Global-MMLU", - hf_subset=standardize_tag(language.value), - evaluation_splits=("test",), - few_shots_split="dev", - hf_filter=partial( - lambda subset, sensitivity_label, x: x["subject"].lower() == subset - and ( - sensitivity_label == "ALL" or sensitivity_label in x["cultural_sensitivity_label"].replace("-", "UNK") - ) - and all(x[f"option_{opt}"] is not None and x[f"option_{opt}"].strip() for opt in "abcd"), - subset, - sensitivity_label, - ), - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbPMINorm()), - ], - ), - ) - for subset in MMLU_SUBSETS - for language in [ - Language.AMHARIC, - Language.ARABIC, - Language.BENGALI, - Language.CHINESE, - Language.CZECH, - Language.GERMAN, - Language.ENGLISH, - Language.SPANISH, - Language.FRENCH, - Language.HEBREW, - Language.HINDI, - Language.INDONESIAN, - Language.ITALIAN, - Language.JAPANESE, - Language.KOREAN, - Language.MALAY, - Language.DUTCH, - Language.NORWEGIAN, - Language.POLISH, - Language.PORTUGUESE, - Language.ROMANIAN, - Language.RUSSIAN, - Language.SERBIAN, - Language.SWEDISH, - Language.SWAHILI, - Language.TAMIL, - Language.TELUGU, - Language.THAI, - Language.TURKISH, - Language.UKRAINIAN, - Language.URDU, - Language.VIETNAMESE, - Language.YORUBA, - Language.ZULU, - ] - for formulation in [ - MCFFormulation(), - ] - for sensitivity_label in ["ALL"] -] diff --git a/src/lighteval/tasks/multilingual/tasks/hellaswag_hin.py b/src/lighteval/tasks/multilingual/tasks/hellaswag_hin.py deleted file mode 100644 index 7d8b2bf50..000000000 --- a/src/lighteval/tasks/multilingual/tasks/hellaswag_hin.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -name: -Hellaswag Hin - -dataset: -ai4bharat/hellaswag-hi - -abstract: -Hellaswag Hin multilingual benchmark. - -languages: -hindi - -tags: -multilingual, multiple-choice, reasoning - -paper: -""" - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.hellaswag import get_hellaswag_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"community_hellaswag_{Language.HINDI.value}_{formulation.name.lower()}", - prompt_function=get_hellaswag_prompt_function( - language=Language.HINDI, - adapter=lambda line: { - "ctx_a": line["ctx_a"], - "continuations": line["endings"], - "gold_idx": int(line["label"]), - }, - formulation=formulation, - ), - hf_repo="ai4bharat/hellaswag-hi", - hf_filter=lambda line: all(len(choice.strip()) > 0 for choice in line["endings"]), - hf_subset="hi", - evaluation_splits=("validation",), - few_shots_split="validation", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - ], - ), - ) - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] -] diff --git a/src/lighteval/tasks/multilingual/tasks/hellaswag_tel.py b/src/lighteval/tasks/multilingual/tasks/hellaswag_tel.py deleted file mode 100644 index a512619ba..000000000 --- a/src/lighteval/tasks/multilingual/tasks/hellaswag_tel.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -name: -Hellaswag Tel - -dataset: -LightFury9/hellaswag-telugu - -abstract: -Hellaswag Tel multilingual benchmark. - -languages: -telugu - -tags: -multilingual, multiple-choice, reasoning - -paper: -""" - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.hellaswag import get_hellaswag_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"community_hellaswag_{Language.TELUGU.value}_{formulation.name.lower()}", - prompt_function=get_hellaswag_prompt_function( - language=Language.TELUGU, - adapter=lambda line: { - "ctx_a": line["ctx_a"], - "continuations": line["endings"], - "gold_idx": int(line["label"]), - }, - formulation=formulation, - ), - hf_repo="LightFury9/hellaswag-telugu", - hf_subset="default", - evaluation_splits=("valid",), - few_shots_split="train", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] -] diff --git a/src/lighteval/tasks/multilingual/tasks/hellaswag_tha.py b/src/lighteval/tasks/multilingual/tasks/hellaswag_tha.py deleted file mode 100644 index ae4c715b3..000000000 --- a/src/lighteval/tasks/multilingual/tasks/hellaswag_tha.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -name: -Hellaswag Tha - -dataset: -lighteval/hellaswag_thai - -abstract: -Hellaswag Thai This is a Thai adaptation of the Hellaswag task. Similar to the -Turkish version, there's no specific paper, but it has been found to be -effective for evaluating Thai language models on commonsense reasoning tasks. - -languages: -thai - -tags: -multilingual, multiple-choice, reasoning - -paper: -""" - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.hellaswag import get_hellaswag_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"community_hellaswag_{Language.THAI.value}_{formulation.name.lower()}", - prompt_function=get_hellaswag_prompt_function( - language=Language.THAI, - adapter=lambda line: { - "ctx_a": line["ctx_a"], - "ctx_b": line["ctx_b"], - "continuations": line["endings"], - "gold_idx": int(line["label"]), - }, - formulation=formulation, - wikihow_artifacts=[" [ชื่อ]", " [ส่วนหัว]", " [ขั้นตอน]", " [header]", " [Header]"], - ), - hf_repo="lighteval/hellaswag_thai", - hf_subset="default", - evaluation_splits=["validation"], - few_shots_split="train", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] -] diff --git a/src/lighteval/tasks/multilingual/tasks/hellaswag_tur.py b/src/lighteval/tasks/multilingual/tasks/hellaswag_tur.py deleted file mode 100644 index 5c0df54b9..000000000 --- a/src/lighteval/tasks/multilingual/tasks/hellaswag_tur.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -name: -Hellaswag Tur - -dataset: -malhajar/hellaswag_tr-v0.2 - -abstract: -Hellaswag Turkish This is a Turkish adaptation of the Hellaswag task. While -there's no specific paper for this version, it has been found to work well for -evaluating Turkish language models on commonsense reasoning tasks. We don't -handle them in single task as there is quite a lot of differences -(dataset/subset, dot replacement, etc.) which would make it hard to read - -languages: -turkish - -tags: -multilingual, multiple-choice, reasoning - -paper: -""" - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.hellaswag import get_hellaswag_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"community_hellaswag_{Language.TURKISH.value}_{formulation.name.lower()}", - prompt_function=get_hellaswag_prompt_function( - language=Language.TURKISH, - adapter=lambda line: { - "ctx_a": line["ctx_a"], - "ctx_b": line["ctx_b"], - "continuations": line["endings"], - "gold_idx": int(line["label"]), - }, - formulation=formulation, - # https://github.com/malhajar17/lm-evaluation-harness_turkish/blob/main/lm_eval/tasks/hellaswag_tr-v0.2/utils.py - wikihow_artifacts=[" [title]", " [başlık]", " [adım]", " [header]"], - ), - hf_repo="malhajar/hellaswag_tr-v0.2", - hf_subset="default", - evaluation_splits=["validation"], - hf_avail_splits=["validation"], - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] -] diff --git a/src/lighteval/tasks/multilingual/tasks/hindi_arc.py b/src/lighteval/tasks/multilingual/tasks/hindi_arc.py deleted file mode 100644 index 337bfafba..000000000 --- a/src/lighteval/tasks/multilingual/tasks/hindi_arc.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -name: -Hindi Arc - -dataset: -ai4bharat/ai2_arc-hi - -abstract: -Hindi Arc multilingual benchmark. - -languages: -hindi - -tags: -multilingual, multiple-choice, reasoning - -paper: -""" - -from string import ascii_uppercase - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbPMINorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"community_arc_{Language.HINDI.value}_{formulation.name.lower()}:{subset}", - prompt_function=get_mcq_prompt_function( - Language.HINDI, - lambda line: { - "question": line["question"], - "choices": line["choices"]["text"], - "gold_idx": int(line["answerKey"]) - 1 - if line["answerKey"].isdigit() - else ascii_uppercase.index(line["answerKey"]), - }, - formulation=formulation, - ), - hf_repo="ai4bharat/ai2_arc-hi", - hf_subset=f"ARC-{subset.capitalize()}", - evaluation_splits=("test",), - few_shots_split="validation", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ] - + ([LogLikelihoodAccMetric(normalization=LogProbPMINorm())] if subset == "challenge" else []), # type: ignore - ), - ) - for subset in ["easy", "challenge"] - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/hindi_boolq.py b/src/lighteval/tasks/multilingual/tasks/hindi_boolq.py deleted file mode 100644 index b007886e3..000000000 --- a/src/lighteval/tasks/multilingual/tasks/hindi_boolq.py +++ /dev/null @@ -1,61 +0,0 @@ -""" -name: -Hindi Boolq - -dataset: -ai4bharat/boolq-hi - -abstract: -Hindi Boolq multilingual benchmark. - -languages: -gujarati, hindi, malayalam, marathi, tamil - -tags: -classification, multilingual, qa - -paper: -""" - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, - MultilingualQuasiExactMatchMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.boolq import get_boolq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"community_boolq_{language.value}", - prompt_function=get_boolq_prompt_function( - language, - lambda line: { - "question": line["question"], - "answer": line["answer"], - "context": line["passage"], - }, - formulation=CFFormulation(), - ), - hf_repo="ai4bharat/boolq-hi", - hf_subset=standardize_tag(language.value), - evaluation_splits=("validation",), - few_shots_split="train", - generation_size=5, - stop_sequence=["\n"], - metrics=[MultilingualQuasiExactMatchMetric(language, "full"), LogLikelihoodAccMetric()], - ) - for language in [ - Language.HINDI, - Language.GUJARATI, - Language.MALAYALAM, - Language.MARATHI, - Language.TAMIL, - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/indicqa.py b/src/lighteval/tasks/multilingual/tasks/indicqa.py deleted file mode 100644 index c3348ce42..000000000 --- a/src/lighteval/tasks/multilingual/tasks/indicqa.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -name: -Indicqa - -dataset: -ai4bharat/IndicQA - -abstract: -IndicQA: A reading comprehension dataset for 11 Indian languages. - -languages: -assamese, bengali, gujarati, hindi, kannada, malayalam, marathi, oriya, punjabi, -tamil, telugu - -tags: -multilingual, qa - -paper: -https://arxiv.org/abs/2407.13522 -""" - -from langcodes import Language as LangCodeLanguage - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"indicqa_{language.value}", - prompt_function=get_qa_prompt_function( - language, - lambda line: { - "question": line["question"], - "context": line["context"], - "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0], - }, - ), - hf_repo="ai4bharat/IndicQA", - hf_subset=f"indicqa.{LangCodeLanguage.get(language.value).language}", - hf_filter=lambda line: any(len(ans) > 0 for ans in line["answers"]["text"]), - hf_revision="92d96092ae229950973dac3b9998f8b3a8949b0a", - evaluation_splits=("test",), - hf_avail_splits=("test",), - generation_size=400, - metrics=( - MultilingualQuasiExactMatchMetric(language, "prefix"), - MultilingualQuasiF1ScoreMetric(language), - ), - stop_sequence=("\n",), - ) - for language in [ - Language.ASSAMESE, - Language.BENGALI, - Language.GUJARATI, - Language.HINDI, - Language.KANNADA, - Language.MALAYALAM, - Language.MARATHI, - Language.ORIYA, - Language.PUNJABI, - Language.TAMIL, - Language.TELUGU, - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/kenswquad.py b/src/lighteval/tasks/multilingual/tasks/kenswquad.py deleted file mode 100644 index 2a0e780dd..000000000 --- a/src/lighteval/tasks/multilingual/tasks/kenswquad.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -name: -Kenswquad - -dataset: -lighteval/KenSwQuAD - -abstract: -KenSwQuAD: A question answering dataset for Kenyan Swahili. - -languages: -swahili - -tags: -multilingual, qa - -paper: -https://arxiv.org/abs/2205.02364 -""" - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"kenswquad_{Language.SWAHILI.value}", - prompt_function=get_qa_prompt_function( - Language.SWAHILI, - lambda line: { - "question": line["question"], - "context": line["context"], - "choices": [line["answer"]], - }, - ), - hf_repo="lighteval/KenSwQuAD", - hf_subset="default", - evaluation_splits=("test",), - few_shots_split="validation", - metrics=( - MultilingualQuasiExactMatchMetric(Language.SWAHILI, "prefix"), - MultilingualQuasiF1ScoreMetric(Language.SWAHILI), - ), - generation_size=400, - stop_sequence=("\n",), - ) -] diff --git a/src/lighteval/tasks/multilingual/tasks/kyrgyz_eval.py b/src/lighteval/tasks/multilingual/tasks/kyrgyz_eval.py deleted file mode 100644 index 3614f3372..000000000 --- a/src/lighteval/tasks/multilingual/tasks/kyrgyz_eval.py +++ /dev/null @@ -1,441 +0,0 @@ -""" -name: -Kyrgyz Evals - -dataset: -TTimur/kyrgyzMMLU, TTimur/kyrgyzRC, TTimur/hellaswag_kg, -TTimur/winogrande_kg, TTimur/truthfulqa_kg, TTimur/gsm8k_kg, TTimur/boolq_kg - -abstract: -Comprehensive evaluation suite for Kyrgyz language understanding, including MMLU, -Reading Comprehension, HellaSwag, Winogrande, TruthfulQA, GSM8K, and BoolQ tasks. - -languages: -kyrgyz - -tags: -knowledge, reading-comprehension, common-sense, reasoning, math, truthfulness - -paper: -https://ieeexplore.ieee.org/document/11206960 -""" - -import json -import string -from functools import partial - -from lighteval.metrics.metrics import Metrics -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -LETTER_INDICES = string.ascii_uppercase - -# ============================================ -# ====== MMLU TASKS ========================== -# ============================================ - -MMLU_SUBSETS = [ - "kyrgyz_mmlu_all", - "kyrgyz_mmlu_history", - "kyrgyz_mmlu_literature", - "kyrgyz_mmlu_medicine", - "kyrgyz_mmlu_lang", - "kyrgyz_mmlu_biology", - "kyrgyz_mmlu_chemistry", - "kyrgyz_mmlu_geography", - "kyrgyz_mmlu_math", - "kyrgyz_mmlu_physics", -] - - -def kyrgyz_mmlu_prompt(line: dict, task_name: str = None) -> Doc: - """ - Creates a prompt for MMLU-style multiple-choice tasks in Kyrgyz. - """ - question = line["Суроо (KG)"] - correct_answer = str(line["Туура жооп"]) - - choices = [line["А (KG)"], line["Б (KG)"], line["В (KG)"], line["Г (KG)"], line["Д (KG)"]] - choices = [c.strip() for c in choices if c] - - letter_to_index = {"а": 0, "б": 1, "в": 2, "г": 3, "д": 4} - gold_index = letter_to_index.get(correct_answer.lower(), 0) - - instruction = "Сиз билимиңизге жана жөндөмүңүзгө жараша суроолорго жооп берген AIсыз. Сизге суроо жана 2-5 жооп варианты берилет, туура жооптун НОМЕРИН (индексин) гана кайтарышыңыз керек.\n\n" - - query = f"{instruction}Суроо: {question}\n\nСунушталган жооптор:\n" - - for i, choice in enumerate(choices): - if choice: - query += f"{i}. {choice}\n" - - query += "\n\nТуура жоопту тандаңыз:" - - return Doc( - task_name=task_name, - query=query, - choices=choices, - gold_index=gold_index, - instruction=instruction, - ) - - -class CustomKyrgyzMMLUTask(LightevalTaskConfig): - def __init__(self, name, hf_subset): - super().__init__( - name=name, - hf_subset=hf_subset, - prompt_function=partial(kyrgyz_mmlu_prompt), - hf_repo="TTimur/kyrgyzMMLU", - metrics=[ - Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()}), - Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbTokenNorm()}), - ], - hf_avail_splits=["test", "validation"], - evaluation_splits=["test"], - few_shots_split="validation", - few_shots_select="sequential", - generation_size=-1, - stop_sequence=None, - version=0, - ) - - -MMLU_TASKS = [CustomKyrgyzMMLUTask(name=f"kyrgyz_evals:{subset}", hf_subset=subset) for subset in MMLU_SUBSETS] - - -# ============================================ -# ====== READING COMPREHENSION TASKS ========= -# ============================================ - -RC_SUBSETS = [ - "kyrgyz_rc_all", - "kyrgyz_rc_literature", - "kyrgyz_rc_math", - "kyrgyz_rc_news", - "kyrgyz_rc_wiki", -] - - -def kyrgyz_rc_prompt(line: dict, task_name: str = None) -> Doc: - """ - Creates a prompt for Reading Comprehension tasks in Kyrgyz. - """ - text = line["Текст (KG)"] - question = line["Суроо (KG)"] - correct_answer = str(line["Туура жооп"]) - - choices = [line["А (KG)"], line["Б (KG)"], line["В (KG)"], line["Г (KG)"]] - choices = [c.strip() for c in choices if c] - - letter_to_index = { - "а": 0, - "б": 1, - "в": 2, - "г": 3, - } - gold_index = letter_to_index.get(correct_answer.lower(), 0) - - instruction = "Сизге бир темага байланыштуу бир нече үзүндү текст берилген. Бардык үзүндүлөрдү кунт коюп окуп, андан кийин төмөндөгү суроолорго жооп бериңиздер. Суроо менен 2-4 жооп варианты берилет, туура жооптун НОМЕРИН (индексин) гана кайтарышыңыз керек.\n\n" - - query = f"{instruction}Текст: {text}\n\nСуроо: {question}\n\nСунушталган жооптор:\n" - - for i, choice in enumerate(choices): - if choice: - query += f"{i}. {choice}\n" - - query += "\n\nТуура жоопту тандаңыз:" - - return Doc( - task_name=task_name, - query=query, - choices=choices, - gold_index=gold_index, - instruction=instruction, - ) - - -class CustomKyrgyzRCTask(LightevalTaskConfig): - def __init__(self, name, hf_subset): - super().__init__( - name=name, - hf_subset=hf_subset, - prompt_function=partial(kyrgyz_rc_prompt), - hf_repo="TTimur/kyrgyzRC", - metrics=[ - Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()}), - # Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbTokenNorm()}), - ], - hf_avail_splits=["test", "validation"], - evaluation_splits=["test"], - few_shots_split="validation", - few_shots_select="sequential", - generation_size=-1, - stop_sequence=None, - # trust_dataset=True, - version=0, - ) - - -RC_TASKS = [CustomKyrgyzRCTask(name=f"kyrgyz_evals:{subset}", hf_subset=subset) for subset in RC_SUBSETS] - - -# ============================================ -# ====== HELLASWAG TASK ====================== -# ============================================ - - -def kyrgyz_hellaswag_prompt(line: dict, task_name: str = None) -> Doc: - """ - Hellaswag-style multiple-choice prompt. - - The Kyrgyz dataset provides: - - ctx_a_kg, ctx_b_kg: context pieces - - activity_label_kg: short description - - endings_kg: list of 4 full candidate endings (strings) - - label: correct ending index in [0, 3] - """ - import ast - - ctx_a_kg = line["ctx_a_kg"] if line["ctx_a_kg"] else "." - ctx_b_kg = line["ctx_b_kg"].capitalize() if line["ctx_b_kg"] else "." - - endings_kg = line.get("endings_kg") - - if isinstance(endings_kg, str): - try: - # Try JSON first - endings_kg = json.loads(endings_kg) - except Exception: - try: - endings_kg = ast.literal_eval(endings_kg) - except Exception: - endings_kg = [endings_kg] - - if not isinstance(endings_kg, list): - endings_kg = [str(endings_kg)] - - endings_kg = endings_kg[:4] - - query = ( - "Төмөндө жалпы түшүнүккө (common sense) байланыштуу бир нече тандоо суроолору (жооптору менен) берилген.\n\n" - ) - query += f"Суроо: {line['activity_label_kg']}: {ctx_a_kg} {ctx_b_kg}\n" - query += "".join([f"{letter}. {choice}\n" for letter, choice in zip(LETTER_INDICES, endings_kg)]) - query += "Туура жоопту тандаңыз: " - - gold_ix = int(line["label"]) if line.get("label", "") != "" else -1 - - return Doc( - task_name=task_name, - query=query, - choices=[f" {letter}" for letter in LETTER_INDICES[: len(endings_kg)]], - gold_index=gold_ix, - instruction="Төмөндө жалпы түшүнүккө (common sense) байланыштуу бир нече тандоо суроолору (жооптору менен) берилген.\n\n", - ) - - -HELLASWAG_TASK = LightevalTaskConfig( - name="kyrgyz_evals:hellaswag_kg", - prompt_function=kyrgyz_hellaswag_prompt, - hf_repo="TTimur/hellaswag_kg", - hf_subset="default", - metrics=[ - Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()}), - Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbTokenNorm()}), - ], - hf_avail_splits=["train", "validation"], - evaluation_splits=["validation"], - few_shots_split="train", - few_shots_select="sequential", - generation_size=-1, - # trust_dataset=True, - version=0, -) - - -# ============================================ -# ====== WINOGRANDE TASK ===================== -# ============================================ - - -def kyrgyz_winogrande_prompt(line: dict, task_name: str = None) -> Doc: - """ - Creates a prompt for Winogrande tasks in Kyrgyz. - """ - query, end_of_target = line["sentence_kg"].split("_") - end_of_target = end_of_target.strip() - - return Doc( - task_name=task_name, - query=query, - choices=[f"{line['option1_kg']} {end_of_target}", f"{line['option2_kg']} {end_of_target}"], - gold_index=int(line["answer"]) - 1 if line["answer"] != "" else -1, - ) - - -WINOGRANDE_TASK = LightevalTaskConfig( - name="kyrgyz_evals:winogrande_kg", - prompt_function=kyrgyz_winogrande_prompt, - hf_repo="TTimur/winogrande_kg", - hf_subset="default", - metrics=[ - Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()}), - ], - hf_avail_splits=["train", "dev"], - evaluation_splits=["dev"], - few_shots_split="train", - few_shots_select="sequential", - generation_size=-1, - version=0, -) - - -# ============================================ -# ====== TRUTHFULQA TASK ===================== -# ============================================ - - -def kyrgyz_truthful_qa_prompt(line: dict, task_name: str = None) -> Doc: - """ - Creates a prompt for TruthfulQA tasks in Kyrgyz. - """ - import ast - - mc1 = line.get("mc1_targets_kg", "{}") - mc2 = line.get("mc2_targets_kg", "{}") - - if isinstance(mc1, str): - try: - mc1 = ast.literal_eval(mc1) - except (ValueError, SyntaxError): - mc1 = {"choices": [], "labels": []} - else: - mc1 = {"choices": [], "labels": []} - - if isinstance(mc2, str): - try: - mc2 = ast.literal_eval(mc2) - except (ValueError, SyntaxError): - mc2 = {"choices": [], "labels": []} - else: - mc2 = {"choices": [], "labels": []} - - choices = [f" {c}" for c in mc1.get("choices", [])] + [f" {c}" for c in mc2.get("choices", [])] - labels = mc1.get("labels", []) + mc2.get("labels", []) - - return Doc( - task_name=task_name, - query=f"Суроо: {line['Question_kg']}\nЖооп:", - choices=choices, - gold_index=[ix for ix, label in enumerate(labels) if label == 1], - specific={"len_mc1": len(mc1.get("choices", []))}, - ) - - -TRUTHFULQA_TASK = LightevalTaskConfig( - name="kyrgyz_evals:truthfulqa_mc_kg", - prompt_function=kyrgyz_truthful_qa_prompt, - hf_repo="TTimur/truthfulqa_kg", - hf_subset="default", - metrics=[Metrics.truthfulqa_mc_metrics], - hf_avail_splits=["test", "validation"], - evaluation_splits=["test"], - few_shots_split="validation", - few_shots_select="sequential", - generation_size=-1, - # trust_dataset=True, - version=0, -) - - -# ============================================ -# ====== GSM8K TASK ========================== -# ============================================ - - -def kyrgyz_gsm8k_prompt(line: dict, task_name: str = None) -> Doc: - """ - Creates a prompt for GSM8K tasks in Kyrgyz. - """ - return Doc( - task_name=task_name, - query=f"Суроо: {line['question_kg']}\nЖооп:", - choices=[f" {line['answer_kg']}"], - gold_index=0, - ) - - -GSM8K_TASK = LightevalTaskConfig( - name="kyrgyz_evals:gsm8k_kg", - prompt_function=kyrgyz_gsm8k_prompt, - hf_repo="TTimur/gsm8k_kg", - hf_subset="default", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split="train", - few_shots_select="random_sampling_from_train", - generation_size=256, - metrics=[ - Metrics.expr_gold_metric, - # MultilingualQuasiExactMatchMetric(Language.KIRGHIZ, "full"), - ], - stop_sequence=["Суроо:"], - version=0, -) - - -# ============================================ -# ====== BOOLQ TASK ========================== -# ============================================ - - -def kyrgyz_boolq_prompt(line: dict, task_name: str = None) -> Doc: - """ - Creates a prompt for BoolQ tasks in Kyrgyz. - """ - question = line["question_kg"][:-1] if line["question_kg"][-2:] == "??" else line["question_kg"] - - return Doc( - task_name=task_name, - query=f"Текст: {line['passage_kg']}\nСуроо: {question}\nЖооп:", - choices=[" ооба", " жок"], - gold_index=["ооба", "жок"].index(line["answer_kg"]), - ) - - -BOOLQ_TASK = LightevalTaskConfig( - name="kyrgyz_evals:boolq_kg", - prompt_function=kyrgyz_boolq_prompt, - hf_repo="TTimur/boolq_kg", - hf_subset="default", - metrics=[ - # Metrics.loglikelihood_acc_norm, - Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()}), - ], - hf_avail_splits=["train", "val"], - evaluation_splits=["val"], - few_shots_split="train", - few_shots_select="sequential", - generation_size=-1, - version=0, -) - - -# ============================================ -# ====== TASKS TABLE ========================= -# ============================================ - -TASKS_TABLE = ( - MMLU_TASKS - + RC_TASKS - + [ - HELLASWAG_TASK, - WINOGRANDE_TASK, - TRUTHFULQA_TASK, - GSM8K_TASK, - BOOLQ_TASK, - ] -) diff --git a/src/lighteval/tasks/multilingual/tasks/m3exams.py b/src/lighteval/tasks/multilingual/tasks/m3exams.py deleted file mode 100644 index addfd203a..000000000 --- a/src/lighteval/tasks/multilingual/tasks/m3exams.py +++ /dev/null @@ -1,84 +0,0 @@ -""" -name: -M3Exams - -dataset: -chiayewken/m3exam - -abstract: -M3Exam: Multitask Multilingual Multimodal Evaluation Benchmark It also contains -a multimodal version but we don't support that Paper: -https://arxiv.org/abs/2306.05179 - -languages: -afrikaans, chinese, english, italian, javanese, portuguese, swahili, thai, -vietnamese - -tags: -knowledge, multilingual, multiple-choice - -paper: -https://arxiv.org/abs/2306.05179 -""" - -from functools import partial - -from langcodes import Language as LangCodeLanguage -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.adapters import ( - get_m3exam_adapter, -) -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"m3exams_{language.value}_{formulation.name.lower()}", - prompt_function=get_mcq_prompt_function( - language, - partial(get_m3exam_adapter, language), - formulation=formulation, - ), - hf_repo="chiayewken/m3exam", - hf_subset=LangCodeLanguage(standardize_tag(language.value)).language_name().lower(), - evaluation_splits=("test",), - few_shots_split="dev", - generation_size=-1, - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for language in [ - Language.AFRIKAANS, - Language.CHINESE, - Language.ENGLISH, - Language.ITALIAN, - Language.JAVANESE, - Language.PORTUGUESE, - Language.SWAHILI, - Language.THAI, - Language.VIETNAMESE, - ] - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/mathlogicqa_rus.py b/src/lighteval/tasks/multilingual/tasks/mathlogicqa_rus.py deleted file mode 100644 index f7a9e14c3..000000000 --- a/src/lighteval/tasks/multilingual/tasks/mathlogicqa_rus.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -name: -Mathlogicqa Rus - -dataset: -ai-forever/MERA - -abstract: -MathLogicQA is a dataset for evaluating mathematical reasoning in language -models. It consists of multiple-choice questions that require logical reasoning -and mathematical problem-solving. This Russian version is part of the MERA -(Multilingual Evaluation of Reasoning Abilities) benchmark. - -languages: -russian - -tags: -math, multilingual, qa, reasoning - -paper: -https://github.com/ai-forever/MERA -""" - -from string import ascii_uppercase - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"mathlogic_qa_{Language.RUSSIAN.value}_{formulation.name.lower()}", - prompt_function=get_mcq_prompt_function( - Language.RUSSIAN, - lambda line: { - "question": line["inputs"]["text"], - "choices": [line["inputs"][f"option_{i.lower()}"] for i in ascii_uppercase[:4]], - "gold_idx": ascii_uppercase.index(line["outputs"]), - }, - formulation=formulation, - ), - hf_repo="ai-forever/MERA", - hf_subset="mathlogicqa", - evaluation_splits=("train",), - hf_avail_splits=["train"], - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [ - CFFormulation(), - MCFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/meta_mmlu.py b/src/lighteval/tasks/multilingual/tasks/meta_mmlu.py deleted file mode 100644 index 2026b00f5..000000000 --- a/src/lighteval/tasks/multilingual/tasks/meta_mmlu.py +++ /dev/null @@ -1,148 +0,0 @@ -""" -name: -Meta Mmlu - -dataset: -meta-llama/Meta-Llama-3.1-8B-Instruct-evals - -abstract: -Meta MMLU: A multilingual version of MMLU (using google translation) - -languages: -french, german, hindi, italian, portuguese, spanish, thai - -tags: -knowledge, multilingual, multiple-choice - -paper: -https://arxiv.org/abs/2407.21783 -""" - -from functools import partial -from string import ascii_uppercase - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbPMINorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -MMLU_SUBSETS = [ - "abstract_algebra", - "anatomy", - "astronomy", - "business_ethics", - "clinical_knowledge", - "college_biology", - "college_chemistry", - "college_computer_science", - "college_mathematics", - "college_medicine", - "college_physics", - "computer_security", - "conceptual_physics", - "econometrics", - "electrical_engineering", - "elementary_mathematics", - "formal_logic", - "global_facts", - "high_school_biology", - "high_school_chemistry", - "high_school_computer_science", - "high_school_european_history", - "high_school_geography", - "high_school_government_and_politics", - "high_school_macroeconomics", - "high_school_mathematics", - "high_school_microeconomics", - "high_school_physics", - "high_school_psychology", - "high_school_statistics", - "high_school_us_history", - "high_school_world_history", - "human_aging", - "human_sexuality", - "international_law", - "jurisprudence", - "logical_fallacies", - "machine_learning", - "management", - "marketing", - "medical_genetics", - "miscellaneous", - "moral_disputes", - "moral_scenarios", - "nutrition", - "philosophy", - "prehistory", - "professional_accounting", - "professional_law", - "professional_medicine", - "professional_psychology", - "public_relations", - "security_studies", - "sociology", - "us_foreign_policy", - "virology", - "world_religions", -] - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"meta_mmlu_{language.value}_{formulation.name.lower()}:{subset}", - prompt_function=get_mcq_prompt_function( - language, - lambda line: { - "question": line["input_question"], - "choices": [v for _, v in sorted(line["input_choice_list"].items(), key=lambda x: x[0])], - "gold_idx": ascii_uppercase.index(line["input_correct_responses"][0]), - }, - formulation=formulation, - ), - hf_repo="meta-llama/Meta-Llama-3.1-8B-Instruct-evals", - hf_subset=f"Llama-3.1-8B-Instruct-evals__multilingual_mmlu_{standardize_tag(language.value)}__details", - hf_filter=partial( - lambda language, subset, line: line["subtask_name"] - == f"mmlu_{standardize_tag(language.value)}_chat.{subset}", - language, - subset, - ), - evaluation_splits=("latest",), - hf_avail_splits=["latest"], - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbPMINorm()), - ], - ), - ) - for subset in MMLU_SUBSETS - for language in [ - Language.GERMAN, - Language.SPANISH, - Language.FRENCH, - Language.HINDI, - Language.ITALIAN, - Language.PORTUGUESE, - Language.THAI, - ] - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/mgsm.py b/src/lighteval/tasks/multilingual/tasks/mgsm.py deleted file mode 100644 index ba5d9a323..000000000 --- a/src/lighteval/tasks/multilingual/tasks/mgsm.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -name: -Mgsm - -dataset: -juletxara/mgsm - -abstract: -Mgsm multilingual benchmark. - -languages: -bengali, chinese, english, french, german, japanese, russian, spanish, swahili, -telugu, thai - -tags: -math, multilingual, reasoning - -paper: -""" - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"mgsm_{language.value}", - prompt_function=get_qa_prompt_function( - language, - lambda line: { - "question": line["question"], - # The cot is available but we have no use: - # line["answer"] - "choices": [str(line["answer_number"])], - }, - ), - hf_repo="juletxara/mgsm", - hf_subset=standardize_tag(language.value), - evaluation_splits=("test",), - few_shots_split="train", - generation_size=25, - metrics=[ - MultilingualQuasiExactMatchMetric(language, "full"), - ], - stop_sequence=("\n",), - ) - for language in [ - Language.ENGLISH, - Language.SPANISH, - Language.FRENCH, - Language.GERMAN, - Language.RUSSIAN, - Language.CHINESE, - Language.JAPANESE, - Language.THAI, - Language.SWAHILI, - Language.BENGALI, - Language.TELUGU, - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/mintaka.py b/src/lighteval/tasks/multilingual/tasks/mintaka.py deleted file mode 100644 index c6ad89cf2..000000000 --- a/src/lighteval/tasks/multilingual/tasks/mintaka.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -name: -Mintaka - -dataset: -AmazonScience/mintaka - -abstract: -Mintaka multilingual benchmark. - -languages: -arabic, english, french, german, hindi, italian, japanese, portuguese, spanish - -tags: -knowledge, multilingual, qa - -paper: -""" - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"mintaka_{lang.value}", - prompt_function=get_qa_prompt_function( - lang, - lambda line: { - "question": line["question"], - "choices": [line["answerText"]], - }, - ), - hf_repo="AmazonScience/mintaka", - hf_subset=standardize_tag(lang.value), - evaluation_splits=("test",), - few_shots_split="train", - generation_size=400, - stop_sequence=("\n",), - metrics=[ - MultilingualQuasiExactMatchMetric(lang, "prefix"), - MultilingualQuasiF1ScoreMetric(lang), - ], - ) - for lang in [ - Language.ARABIC, - Language.GERMAN, - Language.ENGLISH, - Language.SPANISH, - Language.FRENCH, - Language.HINDI, - Language.ITALIAN, - Language.JAPANESE, - Language.PORTUGUESE, - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/mkqa.py b/src/lighteval/tasks/multilingual/tasks/mkqa.py deleted file mode 100644 index 4e27ffecb..000000000 --- a/src/lighteval/tasks/multilingual/tasks/mkqa.py +++ /dev/null @@ -1,107 +0,0 @@ -""" -name: -Mkqa - -dataset: -apple/mkqa - -abstract: -Mkqa multilingual benchmark. - -languages: -arabic, chinese, chinese_hong_kong, chinese_traditional, danish, dutch, english, -finnish, french, german, hebrew, hungarian, italian, japanese, khmer, korean, -malay, norwegian, polish, portuguese, russian, spanish, swedish, thai, turkish, -vietnamese - -tags: -multilingual, qa - -paper: -""" - -from functools import partial - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.adapters import ( - get_mkqa_adapter, -) -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -MKQA_TASK_TO_ID = { - "entity": 0, - "long_answer": 1, - # "unanswerable": 2, - "date": 3, - "number": 4, - "number_with_unit": 5, - "short_phrase": 6, - "binary": 7, -} - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"mkqa_{language.value}:{subset}", - prompt_function=get_qa_prompt_function(language, partial(get_mkqa_adapter, language)), - hf_repo="apple/mkqa", - hf_subset="mkqa", - hf_revision="325131889721ae0ed885b76ecb8011369d75abad", - hf_filter=partial( - lambda language, subset, line: line["answers"][ - "zh_cn" if language == Language.CHINESE else standardize_tag(language.value) - ][0]["type"] - == MKQA_TASK_TO_ID[subset], - language, - subset, - ), - evaluation_splits=("train",), - hf_avail_splits=["train"], - stop_sequence=("\n",), - metrics=[ - MultilingualQuasiExactMatchMetric(language, "prefix"), - MultilingualQuasiF1ScoreMetric(language), - ] - if subset in ["entity", "long_answer", "short_phrase"] - else [ - MultilingualQuasiExactMatchMetric(language, "full"), - ], - ) - for subset in MKQA_TASK_TO_ID.keys() - for language in [ - Language.ARABIC, - Language.DANISH, - Language.GERMAN, - Language.ENGLISH, - Language.SPANISH, - Language.FINNISH, - Language.FRENCH, - Language.HEBREW, - Language.HUNGARIAN, - Language.ITALIAN, - Language.JAPANESE, - Language.KOREAN, - Language.KHMER, - Language.MALAY, - Language.DUTCH, - Language.NORWEGIAN, - Language.POLISH, - Language.PORTUGUESE, - Language.RUSSIAN, - Language.SWEDISH, - Language.THAI, - Language.TURKISH, - Language.VIETNAMESE, - Language.CHINESE, # Simplified - # Language.CHINESE_HONG_KONG, - # Language.CHINESE_TRADITIONAL, - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/mlmm_arc_challenge.py b/src/lighteval/tasks/multilingual/tasks/mlmm_arc_challenge.py deleted file mode 100644 index f7485124f..000000000 --- a/src/lighteval/tasks/multilingual/tasks/mlmm_arc_challenge.py +++ /dev/null @@ -1,110 +0,0 @@ -""" -name: -Mlmm Arc Challenge - -dataset: -jon-tow/okapi_arc_challenge - -abstract: -ARC (AI2 Reasoning Challenge) is a dataset for question answering that requires -reasoning. It consists of multiple-choice science questions from 3rd to 9th -grade exams. The dataset is split into two parts: ARC-Easy and ARC-Challenge. -ARC-Easy contains questions that can be answered correctly by both humans and -simple baseline models. ARC-Challenge contains questions that are difficult for -both humans and current AI systems. Similar to MMLU, ARC tasks uses PMI -normalization by default but only for the challenge set. - -languages: -arabic, bengali, catalan, chinese, croatian, danish, dutch, french, german, -hindi, hungarian, indonesian, italian, kannada, malayalam, marathi, nepali, -romanian, russian, serbian, slovak, spanish, tamil, telugu, ukrainian, -vietnamese - -tags: -multilingual, multiple-choice, reasoning - -paper: -https://github.com/nlp-uoregon/mlmm-evaluation -""" - -from string import ascii_uppercase - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbPMINorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"mlmm_arc_{language.value}_{formulation.name.lower()}:challenge", - prompt_function=get_mcq_prompt_function( - language, - lambda line: { - "question": line["question"], - "choices": line["choices"]["text"], - "gold_idx": int(line["answerKey"]) - 1 - if line["answerKey"].isdigit() - else ascii_uppercase.index(line["answerKey"]), - }, - formulation=formulation, - ), - hf_repo="jon-tow/okapi_arc_challenge", - hf_subset=standardize_tag(language.value), - hf_revision="823d5d7bfaf8974a3ab52a825b6cf4903b35dbc4", - evaluation_splits=("test",), - few_shots_split="train", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbPMINorm()), - ], - ), - ) - for language in [ - Language.RUSSIAN, - Language.GERMAN, - Language.CHINESE, - Language.FRENCH, - Language.SPANISH, - Language.ITALIAN, - Language.DUTCH, - Language.VIETNAMESE, - Language.INDONESIAN, - Language.ARABIC, - Language.HUNGARIAN, - Language.ROMANIAN, - Language.DANISH, - Language.SLOVAK, - Language.UKRAINIAN, - Language.CATALAN, - Language.SERBIAN, - Language.CROATIAN, - Language.HINDI, - Language.BENGALI, - Language.TAMIL, - Language.NEPALI, - Language.MALAYALAM, - Language.MARATHI, - Language.TELUGU, - Language.KANNADA, - ] - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/mlmm_hellaswag.py b/src/lighteval/tasks/multilingual/tasks/mlmm_hellaswag.py deleted file mode 100644 index 71390edc0..000000000 --- a/src/lighteval/tasks/multilingual/tasks/mlmm_hellaswag.py +++ /dev/null @@ -1,107 +0,0 @@ -""" -name: -Mlmm Hellaswag - -dataset: -jon-tow/okapi_hellaswag - -abstract: -Hellaswag is a commonsense reasoning task that requires models to complete a -given scenario with the most plausible ending. It tests the model's ability to -understand and reason about everyday situations and human behavior. -MLMM-Hellaswag: Multilingual adaptation of Hellaswag - -languages: -arabic, armenian, basque, bengali, catalan, chinese, croatian, danish, dutch, -french, german, gujarati, hindi, hungarian, icelandic, indonesian, italian, -kannada, malayalam, marathi, nepali, norwegian, portuguese, romanian, russian, -serbian, slovak, spanish, swedish, tamil, telugu, ukrainian, vietnamese - -tags: -multilingual, multiple-choice, reasoning - -paper: -https://arxiv.org/abs/2306.07610 -""" - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.hellaswag import get_hellaswag_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"mlmm_hellaswag_{lang.value}_{formulation.name.lower()}", - prompt_function=get_hellaswag_prompt_function( - language=lang, - adapter=lambda line: { - # We don't use activity_label as they are not available - "ctx_a": line["ctx_a"], - "ctx_b": line["ctx_b"], - "continuations": line["endings"], - "gold_idx": int(line["label"]), - }, - formulation=formulation, - ), - hf_repo="jon-tow/okapi_hellaswag", - hf_subset=standardize_tag(lang.value), - hf_revision="fb6b5f2b48a6348ae2a9467a6714c3a054fa8d85", - evaluation_splits=["validation"], - hf_avail_splits=["validation"], - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for lang in [ - Language.ARABIC, - Language.BENGALI, - Language.CATALAN, - Language.DANISH, - Language.GERMAN, - Language.SPANISH, - Language.BASQUE, - Language.FRENCH, - Language.GUJARATI, - Language.HINDI, - Language.CROATIAN, - Language.HUNGARIAN, - Language.ARMENIAN, - Language.INDONESIAN, - Language.ICELANDIC, - Language.ITALIAN, - Language.KANNADA, - Language.MALAYALAM, - Language.MARATHI, - Language.NORWEGIAN, - Language.NEPALI, - Language.DUTCH, - Language.PORTUGUESE, - Language.ROMANIAN, - Language.RUSSIAN, - Language.SLOVAK, - Language.SERBIAN, - Language.SWEDISH, - Language.TAMIL, - Language.TELUGU, - Language.UKRAINIAN, - Language.VIETNAMESE, - Language.CHINESE, - ] - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] -] diff --git a/src/lighteval/tasks/multilingual/tasks/mlmm_mmlu.py b/src/lighteval/tasks/multilingual/tasks/mlmm_mmlu.py deleted file mode 100644 index 18993cd5e..000000000 --- a/src/lighteval/tasks/multilingual/tasks/mlmm_mmlu.py +++ /dev/null @@ -1,166 +0,0 @@ -""" -name: -Mlmm Mmlu - -dataset: -jon-tow/okapi_mmlu - -abstract: -MLMM MMLU: Another multilingual version of MMLU - -languages: -arabic, bengali, catalan, chinese, croatian, danish, dutch, french, german, -hindi, hungarian, indonesian, italian, kannada, malayalam, marathi, nepali, -romanian, russian, serbian, slovak, spanish, tamil, telugu, ukrainian, -vietnamese - -tags: -knowledge, multilingual, multiple-choice - -paper: -https://github.com/nlp-uoregon/mlmm-evaluation -""" - -from functools import partial -from string import ascii_uppercase - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbPMINorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -MMLU_SUBSETS = [ - "abstract_algebra", - "anatomy", - "astronomy", - "business_ethics", - "clinical_knowledge", - "college_biology", - "college_chemistry", - "college_computer_science", - "college_mathematics", - "college_medicine", - "college_physics", - "computer_security", - "conceptual_physics", - "econometrics", - "electrical_engineering", - "elementary_mathematics", - "formal_logic", - "global_facts", - "high_school_biology", - "high_school_chemistry", - "high_school_computer_science", - "high_school_european_history", - "high_school_geography", - "high_school_government_and_politics", - "high_school_macroeconomics", - "high_school_mathematics", - "high_school_microeconomics", - "high_school_physics", - "high_school_psychology", - "high_school_statistics", - "high_school_us_history", - "high_school_world_history", - "human_aging", - "human_sexuality", - "international_law", - "jurisprudence", - "logical_fallacies", - "machine_learning", - "management", - "marketing", - "medical_genetics", - "miscellaneous", - "moral_disputes", - "moral_scenarios", - "nutrition", - "philosophy", - "prehistory", - "professional_accounting", - "professional_law", - "professional_medicine", - "professional_psychology", - "public_relations", - "security_studies", - "sociology", - "us_foreign_policy", - "virology", - "world_religions", -] - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"mlmm_mmlu_{language.value}_{formulation.name.lower()}:{subset}", - prompt_function=get_mcq_prompt_function( - language, - lambda line: { - "question": line["question"], - "choices": line["choices"], - "gold_idx": ascii_uppercase.index(line["answer"]), - }, - formulation=formulation, - ), - hf_repo="jon-tow/okapi_mmlu", - hf_subset=standardize_tag(language.value), - hf_revision="refs/pr/1", - hf_filter=partial(lambda subset, line: line["id"].split("/")[0] == subset, subset), - evaluation_splits=("test",), - few_shots_split="dev", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbPMINorm()), - ], - ), - ) - for subset in MMLU_SUBSETS - for language in [ - Language.RUSSIAN, - Language.GERMAN, - Language.CHINESE, - Language.FRENCH, - Language.SPANISH, - Language.ITALIAN, - Language.DUTCH, - Language.VIETNAMESE, - Language.INDONESIAN, - Language.ARABIC, - Language.HUNGARIAN, - Language.ROMANIAN, - Language.DANISH, - Language.SLOVAK, - Language.UKRAINIAN, - Language.CATALAN, - Language.SERBIAN, - Language.CROATIAN, - Language.HINDI, - Language.BENGALI, - Language.TAMIL, - Language.NEPALI, - Language.MALAYALAM, - Language.MARATHI, - Language.TELUGU, - Language.KANNADA, - ] - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/mlmm_truthfulqa.py b/src/lighteval/tasks/multilingual/tasks/mlmm_truthfulqa.py deleted file mode 100644 index bd38d4ae6..000000000 --- a/src/lighteval/tasks/multilingual/tasks/mlmm_truthfulqa.py +++ /dev/null @@ -1,112 +0,0 @@ -""" -name: -Mlmm Truthfulqa - -dataset: -jon-tow/okapi_truthfulqa - -abstract: -TruthfulQA: Measuring How Models Mimic Human Falsehoods - -languages: -arabic, armenian, basque, bengali, catalan, chinese, croatian, danish, dutch, -french, german, gujarati, hindi, hungarian, icelandic, indonesian, italian, -kannada, malayalam, marathi, nepali, norwegian, portuguese, romanian, russian, -serbian, slovak, spanish, swedish, tamil, telugu, ukrainian, vietnamese - -tags: -factuality, multilingual, qa - -paper: -https://arxiv.org/abs/2109.07958 -""" - -from functools import partial - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"mlmm_truthfulqa_{language.value}_{formulation.name.lower()}:{subset}", - prompt_function=get_mcq_prompt_function( - language, - partial( - lambda subset, line: { - "question": line["question"], - "choices": line[f"{subset}_targets"]["choices"], - "gold_idx": [ix for ix, label in enumerate(line[f"{subset}_targets"]["labels"]) if label == 1], # type: ignore - }, - subset, - ), - formulation=formulation, - ), - hf_repo="jon-tow/okapi_truthfulqa", - hf_subset=standardize_tag(language.value), - hf_revision="cdd5db1a66fd04105622109d1c2a5cbc8cde7586", - evaluation_splits=("validation",), - hf_avail_splits=["validation"], - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for subset in ["mc1", "mc2"] - for language in [ - Language.ARABIC, - Language.BENGALI, - Language.CATALAN, - Language.DANISH, - Language.GERMAN, - Language.SPANISH, - Language.BASQUE, - Language.FRENCH, - Language.GUJARATI, - Language.HINDI, - Language.CROATIAN, - Language.HUNGARIAN, - Language.ARMENIAN, - Language.INDONESIAN, - Language.ICELANDIC, - Language.ITALIAN, - Language.KANNADA, - Language.MALAYALAM, - Language.MARATHI, - Language.NORWEGIAN, - Language.NEPALI, - Language.DUTCH, - Language.PORTUGUESE, - Language.ROMANIAN, - Language.RUSSIAN, - Language.SLOVAK, - Language.SERBIAN, - Language.SWEDISH, - Language.TAMIL, - Language.TELUGU, - Language.UKRAINIAN, - Language.VIETNAMESE, - Language.CHINESE, - ] - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/mlqa.py b/src/lighteval/tasks/multilingual/tasks/mlqa.py deleted file mode 100644 index 891c9e681..000000000 --- a/src/lighteval/tasks/multilingual/tasks/mlqa.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -name: -Mlqa - -dataset: -facebook/mlqa - -abstract: -MLQA (MultiLingual Question Answering) is a benchmark dataset for evaluating -cross-lingual question answering performance. It consists of QA instances in 7 -languages: English, Arabic, German, Spanish, Hindi, Vietnamese, and Chinese. The -dataset is derived from the SQuAD v1.1 dataset, with questions and contexts -translated by professional translators. - -languages: -arabic, chinese, german, hindi, spanish, vietnamese - -tags: -multilingual, qa - -paper: -https://arxiv.org/abs/1910.07475 -""" - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"mlqa_{lang.value}", - prompt_function=get_qa_prompt_function( - lang, - lambda line: { - "context": line["context"], - "question": line["question"], - "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0], - }, - ), - hf_repo="facebook/mlqa", - hf_subset=f"mlqa.{standardize_tag(lang.value)}.{standardize_tag(lang.value)}", - hf_revision="397ed406c1a7902140303e7faf60fff35b58d285", - evaluation_splits=("test",), - hf_avail_splits=["test"], - generation_size=400, - stop_sequence=("\n",), - metrics=[ - MultilingualQuasiExactMatchMetric(lang, "prefix"), - MultilingualQuasiF1ScoreMetric(lang), - ], - ) - for lang in [ - Language.ARABIC, - Language.GERMAN, - Language.SPANISH, - Language.CHINESE, - Language.HINDI, - Language.VIETNAMESE, - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/oab_exams.py b/src/lighteval/tasks/multilingual/tasks/oab_exams.py deleted file mode 100644 index a4a28dd5c..000000000 --- a/src/lighteval/tasks/multilingual/tasks/oab_exams.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -name: -Oab Exams - -dataset: -eduagarcia/oab_exams - -abstract: -OAB Exams: A collection of questions from the Brazilian Bar Association exam The -exam is required for anyone who wants to practice law in Brazil - -languages: -portuguese - -tags: -knowledge, multilingual, multiple-choice - -paper: -https://huggingface.co/datasets/eduagarcia/oab_exams -""" - -from string import ascii_uppercase - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"oab_exams_{Language.PORTUGUESE.value}_{formulation.name.lower()}", - prompt_function=get_mcq_prompt_function( - Language.PORTUGUESE, - lambda line: { - "question": line["question"], - "choices": line["choices"]["text"], - "gold_idx": ascii_uppercase.index(line["answerKey"]), - }, - formulation=formulation, - ), - hf_repo="eduagarcia/oab_exams", - hf_subset="default", - evaluation_splits=("train",), - hf_avail_splits=["train"], - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/ocnli.py b/src/lighteval/tasks/multilingual/tasks/ocnli.py deleted file mode 100644 index 17cf4deab..000000000 --- a/src/lighteval/tasks/multilingual/tasks/ocnli.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -name: -Ocnli - -dataset: -clue/clue - -abstract: -Native Chinese NLI dataset based. - -languages: -chinese - -tags: -classification, multilingual, nli - -paper: -https://arxiv.org/pdf/2010.05444 -""" - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.nli import get_nli_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"ocnli_{Language.CHINESE.value}_{formulation.name.lower()}", - prompt_function=get_nli_prompt_function( - language=Language.CHINESE, - adapter=lambda line: { - "premise": line["sentence1"], - "hypothesis": line["sentence2"], - # Since we ignore the neutral label - "gold_idx": {1: 0, 2: 1}[line["label"]], - }, - relations=["entailment", "contradiction"], - formulation=formulation, - ), - hf_repo="clue/clue", - hf_subset="ocnli", - # Only keep the positive and negative examples - hf_filter=lambda x: int(x["label"]) in [1, 2], - evaluation_splits=("validation",), - few_shots_split="train", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=None), - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] -] diff --git a/src/lighteval/tasks/multilingual/tasks/openai_mmlu.py b/src/lighteval/tasks/multilingual/tasks/openai_mmlu.py deleted file mode 100644 index b3254dd09..000000000 --- a/src/lighteval/tasks/multilingual/tasks/openai_mmlu.py +++ /dev/null @@ -1,149 +0,0 @@ -""" -name: -Openai Mmlu - -dataset: -openai/MMMLU - -abstract: -Openai Mmlu multilingual benchmark. - -languages: -arabic, bengali, chinese, french, german, hindi, indonesian, italian, japanese, -korean, portuguese, spanish, swahili, yoruba - -tags: -knowledge, multilingual, multiple-choice - -paper: -""" - -from functools import partial -from string import ascii_uppercase - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbPMINorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -MMLU_SUBSETS = [ - "abstract_algebra", - "anatomy", - "astronomy", - "business_ethics", - "clinical_knowledge", - "college_biology", - "college_chemistry", - "college_computer_science", - "college_mathematics", - "college_medicine", - "college_physics", - "computer_security", - "conceptual_physics", - "econometrics", - "electrical_engineering", - "elementary_mathematics", - "formal_logic", - "global_facts", - "high_school_biology", - "high_school_chemistry", - "high_school_computer_science", - "high_school_european_history", - "high_school_geography", - "high_school_government_and_politics", - "high_school_macroeconomics", - "high_school_mathematics", - "high_school_microeconomics", - "high_school_physics", - "high_school_psychology", - "high_school_statistics", - "high_school_us_history", - "high_school_world_history", - "human_aging", - "human_sexuality", - "international_law", - "jurisprudence", - "logical_fallacies", - "machine_learning", - "management", - "marketing", - "medical_genetics", - "miscellaneous", - "moral_disputes", - "moral_scenarios", - "nutrition", - "philosophy", - "prehistory", - "professional_accounting", - "professional_law", - "professional_medicine", - "professional_psychology", - "public_relations", - "security_studies", - "sociology", - "us_foreign_policy", - "virology", - "world_religions", -] - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"openai_mmlu_{language[0].value}_{formulation.name.lower()}:{subset}", - prompt_function=get_mcq_prompt_function( - language[0], - lambda line: { - "question": line["Question"], - "choices": [line["A"], line["B"], line["C"], line["D"]], - "gold_idx": ascii_uppercase.index(line["Answer"]), - }, - formulation=formulation, - ), - hf_repo="openai/MMMLU", - hf_subset=language[1], - evaluation_splits=("test",), - hf_avail_splits=["test"], - hf_filter=partial(lambda subset, x: x["Subject"].lower() == subset, subset), - hf_revision="038c7808122969ead7456361af05cb8f47d247f8", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbPMINorm()), - ], - ), - ) - for subset in MMLU_SUBSETS - for language in [ - (Language.ARABIC, "AR_XY"), - (Language.BENGALI, "BN_BD"), - (Language.GERMAN, "DE_DE"), - (Language.SPANISH, "ES_LA"), - (Language.FRENCH, "FR_FR"), - (Language.HINDI, "HI_IN"), - (Language.INDONESIAN, "ID_ID"), - (Language.ITALIAN, "IT_IT"), - (Language.JAPANESE, "JA_JP"), - (Language.KOREAN, "KO_KR"), - (Language.PORTUGUESE, "PT_BR"), - (Language.SWAHILI, "SW_KE"), - (Language.YORUBA, "YO_NG"), - (Language.CHINESE, "ZH_CN"), - ] - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/openbook_ara.py b/src/lighteval/tasks/multilingual/tasks/openbook_ara.py deleted file mode 100644 index 945baf250..000000000 --- a/src/lighteval/tasks/multilingual/tasks/openbook_ara.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -name: -Openbook Ara - -dataset: -OALL/AlGhafa-Arabic-LLM-Benchmark-Translated - -abstract: -OpenBookQA: A Question-Answering Dataset for Open-Book Exams OpenBookQA is a -question-answering dataset modeled after open-book exams for assessing human -understanding of a subject. It consists of multiple-choice questions that -require combining facts from a given open book with broad common knowledge. The -task tests language models' ability to leverage provided information and apply -common sense reasoning. - -languages: -arabic - -tags: -multilingual, multiple-choice, reasoning - -paper: -https://arxiv.org/abs/1809.02789 -""" - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.adapters import ( - alghafa_adapter, -) -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"alghafa_openbookqa_{Language.ARABIC.value}_{formulation.name.lower()}", - prompt_function=get_mcq_prompt_function(Language.ARABIC, alghafa_adapter, formulation=formulation), - hf_repo="OALL/AlGhafa-Arabic-LLM-Benchmark-Translated", - hf_subset="openbook_qa_ext_ar", - hf_revision="08663706ee7cab30c4b7dc1bb00042a3227ce1ff", - evaluation_splits=["test"], - few_shots_split="validation", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/openbook_es.py b/src/lighteval/tasks/multilingual/tasks/openbook_es.py deleted file mode 100644 index 3671e608f..000000000 --- a/src/lighteval/tasks/multilingual/tasks/openbook_es.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -name: -Openbook Es - -dataset: -BSC-LT/openbookqa-es - -abstract: -Spanish version of OpenBookQA from BSC Language Technology group - -languages: -spanish - -tags: -multilingual, multiple-choice, reasoning - -paper: -https://huggingface.co/datasets/BSC-LT/openbookqa-es -""" - -from string import ascii_uppercase - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"openbookqa_{Language.SPANISH.value}_{formulation.name.lower()}", - prompt_function=get_mcq_prompt_function( - Language.SPANISH, - lambda line: { - "question": line["question_stem"], - "choices": line["choices"]["text"], - "gold_idx": ascii_uppercase.index(line["answerKey"]), - }, - formulation=formulation, - ), - hf_repo="BSC-LT/openbookqa-es", - hf_subset="default", - evaluation_splits=("test",), - few_shots_split="validation", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/openbook_rus.py b/src/lighteval/tasks/multilingual/tasks/openbook_rus.py deleted file mode 100644 index 06ae9101f..000000000 --- a/src/lighteval/tasks/multilingual/tasks/openbook_rus.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -name: -Openbook Rus - -dataset: -ai-forever/MERA - -abstract: -The Russian version is part of the MERA (Multilingual Enhanced Russian NLP -Architectures) project. - -languages: -russian - -tags: -multilingual, multiple-choice, reasoning - -paper: -https://arxiv.org/abs/2401.04531 -""" - -from string import ascii_uppercase - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"mera_openbookqa_{Language.RUSSIAN.value}_{formulation.name.lower()}", - prompt_function=get_mcq_prompt_function( - Language.RUSSIAN, - lambda line: { - "question": line["inputs"]["question"], - "choices": [line["inputs"][f"option_{i.lower()}"] for i in ascii_uppercase[:4]], - "gold_idx": ascii_uppercase.index(line["outputs"]), - }, - formulation=formulation, - ), - hf_repo="ai-forever/MERA", - hf_subset="ruopenbookqa", - evaluation_splits=("train",), - hf_avail_splits=["train"], - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/oz.py b/src/lighteval/tasks/multilingual/tasks/oz.py deleted file mode 100644 index 929a1c8b4..000000000 --- a/src/lighteval/tasks/multilingual/tasks/oz.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -name: -OZ Serbian Evals - -dataset: -DjMel/oz-eval - -abstract: -OZ Eval (sr. Opšte Znanje Evaluacija) dataset was created for the purposes of -evaluating General Knowledge of LLM models in Serbian language. Data consists -of 1k+ high-quality questions and answers which were used as part of entry exams -at the Faculty of Philosophy and Faculty of Organizational Sciences, University -of Belgrade. The exams test the General Knowledge of students and were used in -the enrollment periods from 2003 to 2024. - -languages: -serbian - -tags: -knowledge, multiple-choice - -paper: -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def prompt_fn_oz_eval_task(line, task_name: str = None): - query_template = """Pitanje: {question}\n - Ponuđeni odgovori: - A. {choice_a} - B. {choice_b} - C. {choice_c} - D. {choice_d} - E. {choice_e} - - Krajnji odgovor:""" - - options = line["options"] - - query = query_template.format( - question=line["questions"], - choice_a=options[0], - choice_b=options[1], - choice_c=options[2], - choice_d=options[3], - choice_e=options[4], - ) - - choices = ["A", "B", "C", "D", "E"] - return Doc( - task_name=task_name, - query=query, - choices=choices, - gold_index=choices.index(line["answer"]), - ) - - -oz_eval_task = LightevalTaskConfig( - name="serbian_evals:oz_task", - prompt_function=prompt_fn_oz_eval_task, - hf_repo="DjMel/oz-eval", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - metrics=[Metrics.loglikelihood_acc], - version=0, -) - - -# STORE YOUR EVALS -TASKS_TABLE = [oz_eval_task] diff --git a/src/lighteval/tasks/multilingual/tasks/parus.py b/src/lighteval/tasks/multilingual/tasks/parus.py deleted file mode 100644 index 980ae2d94..000000000 --- a/src/lighteval/tasks/multilingual/tasks/parus.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -name: -Parus - -dataset: -ai-forever/MERA - -abstract: -PARus: Plausible Alternatives for Russian PARus is the Russian adaptation of the -COPA task, part of the Russian SuperGLUE benchmark. It evaluates common sense -reasoning and causal inference abilities in Russian language models. - -languages: -russian - -tags: -multilingual - -paper: -https://russiansuperglue.com/tasks/task_info/PARus -""" - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.copa import get_copa_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"parus_{Language.RUSSIAN.value}_{formulation.name.lower()}", - prompt_function=get_copa_prompt_function( - language=Language.RUSSIAN, - adapter=lambda line: { - "context": line["inputs"]["premise"], - "cause_effect": line["meta"]["task"], - "continuations": [line["inputs"]["choice1"], line["inputs"]["choice2"]], - "gold_idx": int(line["outputs"]) - 1, - }, - formulation=formulation, - ), - hf_repo="ai-forever/MERA", - hf_subset="parus", - evaluation_splits=["train"], - few_shots_split="validation", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] -] diff --git a/src/lighteval/tasks/multilingual/tasks/paws_x.py b/src/lighteval/tasks/multilingual/tasks/paws_x.py deleted file mode 100644 index 51ff0cdb8..000000000 --- a/src/lighteval/tasks/multilingual/tasks/paws_x.py +++ /dev/null @@ -1,78 +0,0 @@ -""" -name: -Paws X - -dataset: -google-research-datasets/paws-x - -abstract: -PAWS-X: A Cross-lingual Adversarial Dataset for Paraphrase Identification This -dataset contains paraphrase identification pairs in multiple languages. It's -derived from PAWS (Paraphrase Adversaries from Word Scrambling) and We treat -paraphrase as entailment and non-paraphrase as contradiction - -languages: -chinese, english, french, german, japanese, korean, spanish - -tags: -classification, multilingual, nli - -paper: -https://arxiv.org/abs/1908.11828 -""" - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.nli import get_nli_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"pawsx_{language.value}_{formulation.name.lower()}", - prompt_function=get_nli_prompt_function( - language=language, - adapter=lambda line: { - "premise": line["sentence1"], - "hypothesis": line["sentence2"], - # Since we ignore the neutral label - "gold_idx": int(line["label"]), - }, - relations=["entailment", "contradiction"], - formulation=formulation, - ), - hf_repo="google-research-datasets/paws-x", - hf_subset=standardize_tag(language.value), - evaluation_splits=("test",), - few_shots_split="train", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=None), - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for language in [ - Language.GERMAN, - Language.ENGLISH, - Language.SPANISH, - Language.FRENCH, - Language.JAPANESE, - Language.KOREAN, - Language.CHINESE, - ] - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] -] diff --git a/src/lighteval/tasks/multilingual/tasks/piqa_ar.py b/src/lighteval/tasks/multilingual/tasks/piqa_ar.py deleted file mode 100644 index 4dff5978a..000000000 --- a/src/lighteval/tasks/multilingual/tasks/piqa_ar.py +++ /dev/null @@ -1,65 +0,0 @@ -""" -name: -Piqa Ar - -dataset: -OALL/AlGhafa-Arabic-LLM-Benchmark-Translated - -abstract: -PIQA: Physical Interaction Question Answering PIQA is a benchmark for testing -physical commonsense reasoning. This Arabic version is a translation of the -original PIQA dataset, adapted for Arabic language evaluation. It tests the -ability to reason about physical interactions in everyday situations. - -languages: -arabic - -tags: -multilingual, multiple-choice, qa, reasoning - -paper: -https://arxiv.org/abs/1911.11641 -""" - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.adapters import ( - alghafa_adapter, -) -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"alghafa_piqa_{Language.ARABIC.value}_{formulation.name.lower()}", - prompt_function=get_mcq_prompt_function(Language.ARABIC, alghafa_adapter, formulation=formulation), - hf_repo="OALL/AlGhafa-Arabic-LLM-Benchmark-Translated", - hf_revision="08663706ee7cab30c4b7dc1bb00042a3227ce1ff", - hf_subset="piqa_ar", - hf_avail_splits=["test", "validation"], - evaluation_splits=["test"], - few_shots_split="validation", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/rcb.py b/src/lighteval/tasks/multilingual/tasks/rcb.py deleted file mode 100644 index a2d98c288..000000000 --- a/src/lighteval/tasks/multilingual/tasks/rcb.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -name: -Rcb - -dataset: -ai-forever/MERA - -abstract: -Russian Commitment Bank (RCB) is a large-scale NLI dataset with Russian -sentences, collected from the web and crowdsourcing. - -languages: -russian - -tags: -classification, multilingual, nli - -paper: -https://arxiv.org/abs/2401.04531 -""" - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.nli import get_nli_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"rcb_{Language.RUSSIAN.value}_{formulation.name.lower()}", - prompt_function=get_nli_prompt_function( - language=Language.RUSSIAN, - adapter=lambda line: { - "premise": line["inputs"]["premise"], - "hypothesis": line["inputs"]["hypothesis"], - # Since we ignore the neutral label - "gold_idx": int(line["outputs"]) - 1, - }, - relations=["entailment", "contradiction"], - formulation=formulation, - ), - hf_repo="ai-forever/MERA", - hf_subset="rcb", - # Ignore neutral label - hf_filter=lambda x: int(x["outputs"] or "0") in [1, 2], - evaluation_splits=("train",), - few_shots_split="validation", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=None), - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] -] diff --git a/src/lighteval/tasks/multilingual/tasks/sber_squad.py b/src/lighteval/tasks/multilingual/tasks/sber_squad.py deleted file mode 100644 index e9bb58fa1..000000000 --- a/src/lighteval/tasks/multilingual/tasks/sber_squad.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -name: -Sber Squad - -dataset: -kuznetsoffandrey/sberquad - -abstract: -SberQuAD: A large-scale Russian reading comprehension dataset. - -languages: -russian - -tags: -multilingual, qa - -paper: -https://arxiv.org/abs/1912.09723 -""" - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"sber_squad_{Language.RUSSIAN.value}", - prompt_function=get_qa_prompt_function( - Language.RUSSIAN, - lambda line: { - "question": line["question"], - "context": line["context"], - "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0], - }, - ), - hf_repo="kuznetsoffandrey/sberquad", - hf_subset="sberquad", - evaluation_splits=("validation",), - few_shots_split="train", - metrics=( - MultilingualQuasiExactMatchMetric(Language.RUSSIAN, "prefix"), - MultilingualQuasiF1ScoreMetric(Language.RUSSIAN), - ), - generation_size=400, - stop_sequence=("\n",), - ) -] diff --git a/src/lighteval/tasks/multilingual/tasks/serbian_eval.py b/src/lighteval/tasks/multilingual/tasks/serbian_eval.py deleted file mode 100644 index cb3f54177..000000000 --- a/src/lighteval/tasks/multilingual/tasks/serbian_eval.py +++ /dev/null @@ -1,764 +0,0 @@ -""" -name: -Serbian Evals - -dataset: -datatab/serbian-llm-benchmark - -abstract: -The tasks cover a variety of benchmarks, including: standard task like ARC[E][C], -BoolQ, Hellaswag, OpenBookQA,PIQA, Winogrande and a custom OZ Eval. -MMLU is separated by subject and also all in one. - -languages: -serbian - -tags: -knowledge, multiple-choice - -paper: -""" - -from enum import Enum -from typing import List, Optional - -from lighteval.metrics.metrics import Metrics -from lighteval.metrics.normalizations import LogProbCharNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -class HFSubsets(Enum): - """Enum for all available Hugging Face dataset subsets in Serbian evaluation tasks.""" - - HF_BASE_REPO = "datatab/serbian-llm-benchmark" - HF_REVISION = "209c5b5f999cae5c02eef5735eb817ead18ac214" - - # ARC (AI2 Reasoning Challenge) - ARC_EASY = "arc_easy_serbian" - ARC_CHALLENGE = "arc_challenge_serbian" - # Question Answering and Knowledge - BOOLQ = "boolq_serbian" - OPENBOOK = "openbookq_serbian" - # Commonsense Reasoning - HELLASWAG = "hellaswag_serbian" - PIQA = "piqa_serbian" - WINOGRANDE = "winogrande_serbian" - # Custom/Other Task - OZ_EVAL = "oz_eval_serbian" - # MMLU (Miscellaneous) - MMLU_ANATOMY = "mmlu_anatomija_serbian" - MMLU_ASTRONOMY = "mmlu_astronomija_serbian" - MMLU_BUSINESS_ETHICS = "mmlu_poslovna_etika_serbian" - MMLU_CLINICAL_KNOWLEDGE = "mmlu_kliničko_znanje_serbian" - MMLU_MISCELLANEOUS = "mmlu_miscellaneous_serbian" - MMLU_ELECTRONIC_ENGINEERING = "mmlu_electrical_engineering_serbian" - # MMLU (Business Professional) - MMLU_MARKETING = "mmlu_marketing_serbian" - MMLU_MANAGEMENT = "mmlu_management_serbian" - # MMLU (College Level Tasks) - MMLU_COLLEGE_BIOLOGY = "mmlu_college_biology_serbian" - MMLU_COLLEGE_CHEMISTRY = "mmlu_college_chemistry_serbian" - MMLU_COLLEGE_COMPUTER_SCIENCE = "mmlu_college_computer_science_serbian" - MMLU_COLLEGE_MATHEMATICS = "mmlu_college_mathematics_serbian" - MMLU_COLLEGE_MEDICINE = "mmlu_college_medicine_serbian" - MMLU_COLLEGE_PHYSICS = "mmlu_college_physics_serbian" - MMLU_COLLEGE_COMPUTER_SECURITY = "mmlu_computer_security_serbian" - # MMLU (Ethics, Philosophy) - MMLU_MORAL_DISPUTES = "mmlu_moral_disputes_serbian" - MMLU_MORAL_SCENARIOS = "mmlu_moral_scenarios_serbian" - MMLU_PHILOSOPHY = "mmlu_philosophy_serbian" - MMLU_WORLD_RELIGIONS = "mmlu_world_religions_serbian" - # MMLU (High School Level Tasks) - MMLU_HIGH_SCHOOL_BIOLOGY = "mmlu_high_school_biology_serbian" - MMLU_HIGH_SCHOOL_CHEMISTRY = "mmlu_high_school_chemistry_serbian" - MMLU_HIGH_SCHOOL_COMPUTER_SCIENCE = "mmlu_high_school_computer_science_serbian" - MMLU_HIGH_SCHOOL_EURO_HISTORY = "mmlu_high_school_european_history_serbian" - MMLU_HIGH_SCHOOL_GEOGRAPHY = "mmlu_high_school_geography_serbian" - MMLU_HIGH_SCHOOL_MATHEMATICS = "mmlu_high_school_mathematics_serbian" - MMLU_HIGH_SCHOOL_MICROECONOMICS = "mmlu_high_school_microeconomics_serbian" - MMLU_HIGH_SCHOOL_PHYSICS = "mmlu_high_school_physics_serbian" - MMLU_HIGH_SCHOOL_PSYCHOLOGY = "mmlu_high_school_psychology_serbian" - MMLU_HIGH_SCHOOL_STATISTICS = "mmlu_high_school_statistics_serbian" - MMLU_HIGH_SCHOOL_WORLD_HISTORY = "mmlu_high_school_world_history" - # MMLU (Math, Logic) - MMLU_ABSTRACT_ALGEBRA = "mmlu_abstract_algebra_serbian" - MMLU_ELEMENTARY_MATHEMATICS = "mmlu_osnovna_matematika_serbian" - MMLU_FORMAL_LOGIC = "mmlu_formalna_logika_serbian" - MMLU_CONCEPTUAL_PHYSICS = "mmlu_conceptual_physics_serbian" - MMLU_ECONOMETRICS = "mmlu_econometrics_serbian" - MMLU_MACHINE_LEARNING = "mmlu_machine_learning_serbian" - # MMLU (Social Sciences) - MMLU_GLOBAL_FACT = "mmlu_global_facts_serbian" - MMLU_LOGICAL_FALLACIES = "mmlu_logicke_zablude_serbian" - MMLU_SOCIOLOGY = "mmlu_sociology_serbian" - MMLU_HUMAN_AGING = "mmlu_human_aging_serbian" - # MMLU (All-inclusive Task Entry) - MMLU_SERBIAN_ALL = "mmlu_all_serbian" - - -def prompt_fn_oz_eval_task(line, task_name: str = None): - """ - Prepares a question and answer set in Serbian from the OZ Eval (Opšte Znanje Evaluacija) dataset - for use in a LightEval task. This dataset, specifically designed for evaluating general knowledge - in Serbian, contains questions derived from entrance exams at the University of Belgrade's Faculty - of Philosophy and Faculty of Organizational Sciences, covering enrollment periods from 2003 to 2024. - - The function accepts a dictionary with a question, five answer choices, and a correct answer - designation, returning a structured `Doc` object formatted for LightEval's TASKS_TABLE or TASKS_GROUPS. - - Args: - line (dict): A dictionary with required keys: - - 'query' (str): The main question string. - - 'choices' (list of str): A list containing exactly five answer options. - - 'answer_str' (str): A single character from "A" to "E" representing the correct answer. - task_name (str, optional): An optional string specifying the evaluation task name. - - Returns: - Doc: A structured object for LightEval containing: - - task_name (str): The task name, if provided. - - query (str): Formatted question with embedded answer choices. - - choices (list of str): List of option identifiers ["A", "B", "C", "D", "E"]. - - gold_index (int): Index of the correct answer within the 'choices' list. - - Note: - The OZ Eval dataset is available at https://huggingface.co/datasets/DjMel/oz-eval. - - """ - query_template = """Pitanje: {question}\n - Ponuđeni odgovori: - A. {choice_a} - B. {choice_b} - C. {choice_c} - D. {choice_d} - E. {choice_e} - - Krajnji odgovor:""" - - options = line["choices"] - - query = query_template.format( - question=line["query"], - choice_a=options[0], - choice_b=options[1], - choice_c=options[2], - choice_d=options[3], - choice_e=options[4], - ) - - choices = ["A", "B", "C", "D", "E"] - return Doc( - task_name=task_name, - query=query, - choices=choices, - gold_index=choices.index(line["answer_str"]), - ) - - -def serbian_eval_prompt(line: dict, task_name: Optional[str] = None) -> Doc: - """ - Creates a prompt for a multiple-choice task in Serbian. This function formats the prompt - based on the provided query and choices, handling both standard tasks and MMLU-specific - tasks (if "mmlu" is part of the task name). - - The prompt includes an instruction in Serbian, followed by the query, available choices, - and finally the correct answer. The function determines how to compute the correct answer - based on whether the task name contains "mmlu". - - Args: - line (dict): A dictionary containing the following keys: - - "query" (str): The question or query to present to the user. - - "choices" (list of str): A list of possible answer choices. - - "answer" (int or str): The correct answer, either as an index (for regular tasks) - or as a string (for MMLU tasks). - task_name (Optional[str]): The name of the task. If "mmlu" is in the task name, the - function treats the task as an MMLU task and searches for the correct answer - by matching the string value of the answer. - - Returns: - Doc: A `Doc` object containing the formatted prompt, choices, and the correct answer index. - The `Doc` object includes the following fields: - - task_name (Optional[str]): The name of the task. - - query (str): The formatted query prompt in Serbian, including instructions and choices. - - choices (list of str): The list of available answer choices. - - gold_index (int): The index of the correct answer. - - instruction (str): The instruction shown to the user in Serbian. - """ - - question = line["query"] - choices = line["choices"] - instruction = "Na osnovu sledećeg pitanja, izaberite tačanu opciju iz ponuđenih odgovora.\n" - - # Build the query and determine the gold_index in a single pass - query = f"{instruction}Pitanje: {question}\n\nPonuđeni odgovori:\n" - - gold_index = None - - # ARC is base gold index, but MMLU we handle gold index as - if task_name and "mmlu" in task_name: - correct_answer = str(line["answer"]) - gold_index = next((i for i, choice in enumerate(choices) if correct_answer in choice), None) - else: - gold_index = int(line["answer"]) - - # Show all choises - for i, choice in enumerate(choices): - query += f"{i}. {choice}\n" - - query += "\n\nKrajnji odgovor:" - - return Doc( - task_name=task_name, - query=query, - choices=choices, - gold_index=gold_index, - instruction=instruction, - ) - - -def boolq_serbian(line, task_name: str = None): - # remove extra `?` - question = line["question"][:-1] if line["question"][-2:] == "??" else line["question"] - return Doc( - task_name=task_name, - query=f"Passage: {line['passage']}\nQuestion: {question}\nAnswer:", - choices=[" Da", " Ne"], - gold_index=["Da", "Ne"].index(line["answer"]), - ) - - -def create_task_config( - task_name: str, - prompt_function, - hf_repo: str, - hf_subset: str, - metrics: List, - evaluation_splits: List[str] = ["test"], - hf_avail_splits: List[str] = ["test", "validation"], - few_shots_split: str = "validation", - generation_size=5, -) -> LightevalTaskConfig: - """ - Creates a task configuration using dependency injection for flexible task creation. - - Args: - task_name: The name of the task. - prompt_function: The function to generate task prompts. - hf_repo: Hugging Face repository. - hf_subset: Subset of the dataset. - metrics: The metrics to use for the task. - evaluation_splits: The evaluation splits to use (default is "test"). - hf_avail_splits: Available splits (default is "test", "validation"). - few_shots_split: Split used for few-shot examples. - generation_size: Number of generations to produce (default is 5). - - Returns: - A `LightevalTaskConfig` object for the task configuration. - """ - return LightevalTaskConfig( - name=task_name, - prompt_function=prompt_function, - hf_repo=hf_repo, - hf_subset=hf_subset, - hf_avail_splits=hf_avail_splits, - evaluation_splits=evaluation_splits, - few_shots_split=few_shots_split, - few_shots_select="sequential", - metrics=metrics, - generation_size=generation_size, - hf_revision=HFSubsets.HF_REVISION.value, - version=0, - ) - - -# ============================================ -# ===== ARC (AI2 Reasoning Challenge)========= -# ============================================ - -arc_easy = create_task_config( - task_name="serbian_evals:arc_easy", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.ARC_EASY.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -arc_challenge = create_task_config( - task_name="serbian_evals:arc_challenge", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.ARC_CHALLENGE.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -# ============================================ -# ========= Commonsense Reasoning ============ -# ============================================ - -hellaswag = create_task_config( - task_name="serbian_evals:hellaswag", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.HELLASWAG.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) -piqa = create_task_config( - task_name="serbian_evals:piqa", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.PIQA.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -winogrande = create_task_config( - task_name="serbian_evals:winogrande", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.WINOGRANDE.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -# ============================================ -# =========== Custom/Other Task ============== -# ============================================ - -oz_eval = create_task_config( - task_name="serbian_evals:oz_eval", - prompt_function=prompt_fn_oz_eval_task, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.OZ_EVAL.value, - metrics=[Metrics.loglikelihood_acc], -) - -# ============================================ -# ========== MMLU (Miscellaneous) ============ -# ============================================ - -mmlu_anatomy = create_task_config( - task_name="serbian_evals:mmlu_anatomija", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_ANATOMY.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_astronomy = create_task_config( - task_name="serbian_evals:mmlu_astronomija", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_ASTRONOMY.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_business_ethics = create_task_config( - task_name="serbian_evals:mmlu_poslovna_etika", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_BUSINESS_ETHICS.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_clinical_knowledge = create_task_config( - task_name="serbian_evals:mmlu_kliničko_znanje", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_CLINICAL_KNOWLEDGE.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_miscellaneous = create_task_config( - task_name="serbian_evals:mmlu_razno", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_MISCELLANEOUS.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_electrical_engineering = create_task_config( - task_name="serbian_evals:mmlu_elektrotehnika", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_ELECTRONIC_ENGINEERING.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -# ============================================ -# ====== MMLU (All-inclusive Task Entry) ===== -# ============================================ - -mmlu_all = create_task_config( - task_name="serbian_evals:mmlu", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_SERBIAN_ALL.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -# ============================================ -# ======= MMLU (Business Professional) ======= -# ============================================ - -mmlu_marketing = create_task_config( - task_name="serbian_evals:mmlu_marketing", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_MARKETING.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_management = create_task_config( - task_name="serbian_evals:mmlu_manadzment", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_MANAGEMENT.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -# ============================================ -# ======== MMLU (College Level Tasks) ======== -# ============================================ - -mmlu_college_biology = create_task_config( - task_name="serbian_evals:mmlu_fakultet_biologija", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_COLLEGE_BIOLOGY.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_college_chemistry = create_task_config( - task_name="serbian_evals:mmlu_fakultet_hemija", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_COLLEGE_CHEMISTRY.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_college_computer_science = create_task_config( - task_name="serbian_evals:mmlu_fakultet_racunari", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_COLLEGE_COMPUTER_SCIENCE.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_college_mathematics = create_task_config( - task_name="serbian_evals:mmlu_fakultet_matematika", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_COLLEGE_MATHEMATICS.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_college_medicine = create_task_config( - task_name="serbian_evals:mmlu_fakultet_medicina", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_COLLEGE_MEDICINE.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_college_physics = create_task_config( - task_name="serbian_evals:mmlu_fakultet_fizika", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_COLLEGE_PHYSICS.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_computer_security = create_task_config( - task_name="serbian_evals:mmlu_sigurnost_racunara", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_COLLEGE_COMPUTER_SECURITY.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -# ============================================ -# ======== MMLU (Ethics, Philosophy) ========= -# ============================================ - -mmlu_moral_disputes = create_task_config( - task_name="serbian_evals:mmlu_moralni_sporovi", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_MORAL_DISPUTES.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_moral_scenarios = create_task_config( - task_name="serbian_evals:mmlu_moralne_dileme", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_MORAL_SCENARIOS.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_philosophy = create_task_config( - task_name="serbian_evals:mmlu_filozofija", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_PHILOSOPHY.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_world_religions = create_task_config( - task_name="serbian_evals:mmlu_svetska_religija", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_WORLD_RELIGIONS.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -# ============================================ -# ====== MMLU (High School Level Tasks) ====== -# ============================================ - -mmlu_high_school_biology = create_task_config( - task_name="serbian_evals:mmlu_srednja_skola_biologija", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_HIGH_SCHOOL_BIOLOGY.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_high_school_chemistry = create_task_config( - task_name="serbian_evals:mmlu_srednja_skola_hemija", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_HIGH_SCHOOL_CHEMISTRY.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_high_school_computer_science = create_task_config( - task_name="serbian_evals:mmlu_srednja_skola_racunari", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_HIGH_SCHOOL_COMPUTER_SCIENCE.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_high_school_european_history = create_task_config( - task_name="serbian_evals:mmlu_srednja_skola_istorija_evrope", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_HIGH_SCHOOL_EURO_HISTORY.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_high_school_geography = create_task_config( - task_name="serbian_evals:mmlu_srednja_skola_geografija", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_HIGH_SCHOOL_GEOGRAPHY.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_high_school_mathematics = create_task_config( - task_name="serbian_evals:mmlu_srednja_skola_matematika", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_HIGH_SCHOOL_MATHEMATICS.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_high_school_microeconomics = create_task_config( - task_name="serbian_evals:mmlu_srednja_skola_mikroekonomija", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_HIGH_SCHOOL_MICROECONOMICS.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_high_school_physics = create_task_config( - task_name="serbian_evals:mmlu_srednja_skola_fizika", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_HIGH_SCHOOL_PHYSICS.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_high_school_psychology = create_task_config( - task_name="serbian_evals:mmlu_srednja_skola_psihologija", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_HIGH_SCHOOL_PSYCHOLOGY.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_high_school_statistics = create_task_config( - task_name="serbian_evals:mmlu_srednja_skola_statistika", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_HIGH_SCHOOL_STATISTICS.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_high_school_world_history = create_task_config( - task_name="serbian_evals:mmlu_srednja_skola_svetska_istorija", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_HIGH_SCHOOL_WORLD_HISTORY.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -# ============================================ -# ============ MMLU (Math, Logic) ============ -# ============================================ - -mmlu_abstract_algebra = create_task_config( - task_name="serbian_evals:mmlu_abstract_algebra", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_ABSTRACT_ALGEBRA.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_elementary_mathematics = create_task_config( - task_name="serbian_evals:mmlu_osnovna_matematika", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_ELEMENTARY_MATHEMATICS.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_formal_logic = create_task_config( - task_name="serbian_evals:mmlu_formalna_logika", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_FORMAL_LOGIC.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_conceptual_physics = create_task_config( - task_name="serbian_evals:mmlu_konceptualna_fizika", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_CONCEPTUAL_PHYSICS.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_econometrics = create_task_config( - task_name="serbian_evals:mmlu_metrika_ekonomije", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_ECONOMETRICS.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_machine_learning = create_task_config( - task_name="serbian_evals:mmlu_masinsko_ucenje", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_MACHINE_LEARNING.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -# ============================================ -# ========== MMLU (Social Sciences) ========== -# ============================================ - -mmlu_global_facts = create_task_config( - task_name="serbian_evals:mmlu_globalne_cinjenice", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_GLOBAL_FACT.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_logical_fallacies = create_task_config( - task_name="serbian_evals:mmlu_logicke_zablude", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_LOGICAL_FALLACIES.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_sociology = create_task_config( - task_name="serbian_evals:mmlu_sociologija", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_SOCIOLOGY.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -mmlu_human_aging = create_task_config( - task_name="serbian_evals:mmlu_human_aging", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.MMLU_HUMAN_AGING.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -# ============================================ -# ===== Question Answering and Knowledge ===== -# ============================================ - -boolq = create_task_config( - task_name="serbian_evals:boolq", - prompt_function=boolq_serbian, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.BOOLQ.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - -openbook_qa = create_task_config( - task_name="serbian_evals:openbook", - prompt_function=serbian_eval_prompt, - hf_repo=HFSubsets.HF_BASE_REPO.value, - hf_subset=HFSubsets.OPENBOOK.value, - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], -) - - -TASKS_TABLE = [ - arc_easy, - arc_challenge, - boolq, - hellaswag, - openbook_qa, - piqa, - oz_eval, - winogrande, - mmlu_abstract_algebra, - mmlu_anatomy, - mmlu_astronomy, - mmlu_business_ethics, - mmlu_clinical_knowledge, - mmlu_college_biology, - mmlu_college_chemistry, - mmlu_college_computer_science, - mmlu_college_mathematics, - mmlu_college_medicine, - mmlu_college_physics, - mmlu_computer_security, - mmlu_conceptual_physics, - mmlu_econometrics, - mmlu_electrical_engineering, - mmlu_elementary_mathematics, - mmlu_formal_logic, - mmlu_global_facts, - mmlu_high_school_biology, - mmlu_high_school_chemistry, - mmlu_high_school_computer_science, - mmlu_high_school_european_history, - mmlu_high_school_geography, - mmlu_high_school_mathematics, - mmlu_high_school_microeconomics, - mmlu_high_school_physics, - mmlu_high_school_psychology, - mmlu_high_school_statistics, - mmlu_high_school_world_history, - mmlu_human_aging, - mmlu_logical_fallacies, - mmlu_marketing, - mmlu_machine_learning, - mmlu_management, - mmlu_moral_disputes, - mmlu_miscellaneous, - mmlu_moral_scenarios, - mmlu_sociology, - mmlu_philosophy, - mmlu_world_religions, - mmlu_all, -] diff --git a/src/lighteval/tasks/multilingual/tasks/soqal.py b/src/lighteval/tasks/multilingual/tasks/soqal.py deleted file mode 100644 index b60e2220d..000000000 --- a/src/lighteval/tasks/multilingual/tasks/soqal.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -name: -Soqal - -dataset: -OALL/AlGhafa-Arabic-LLM-Benchmark-Native - -abstract: -SOQAL: A large-scale Arabic reading comprehension dataset. - -languages: -arabic - -tags: -multilingual, qa - -paper: -https://arxiv.org/abs/1906.05394 -""" - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.adapters import ( - alghafa_adapter, -) -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"soqal_{Language.ARABIC.value}_{formulation.name.lower()}", - hf_subset="multiple_choice_grounded_statement_soqal_task", - prompt_function=get_mcq_prompt_function(Language.ARABIC, alghafa_adapter, formulation=formulation), - evaluation_splits=["test"], - few_shots_split="validation", - hf_repo="OALL/AlGhafa-Arabic-LLM-Benchmark-Native", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/squad_es.py b/src/lighteval/tasks/multilingual/tasks/squad_es.py deleted file mode 100644 index 7a3172281..000000000 --- a/src/lighteval/tasks/multilingual/tasks/squad_es.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -name: -Squad Es - -dataset: -ccasimiro/squad_es - -abstract: -SQuAD-es: Spanish translation of the Stanford Question Answering Dataset - -languages: -spanish - -tags: -multilingual, qa - -paper: -https://huggingface.co/datasets/ccasimiro/squad_es -""" - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"squad_{Language.SPANISH.value}", - prompt_function=get_qa_prompt_function( - Language.SPANISH, - lambda line: { - "question": line["question"], - "context": line["context"], - "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0], - }, - ), - hf_repo="ccasimiro/squad_es", - hf_subset="v2.0.0", - hf_filter=lambda line: any(len(ans) > 0 for ans in line["answers"]["text"]), - evaluation_splits=("validation",), - few_shots_split="train", - metrics=( - MultilingualQuasiExactMatchMetric(Language.SPANISH, "prefix"), - MultilingualQuasiF1ScoreMetric(Language.SPANISH), - ), - generation_size=400, - stop_sequence=("\n",), - ) -] diff --git a/src/lighteval/tasks/multilingual/tasks/squad_it.py b/src/lighteval/tasks/multilingual/tasks/squad_it.py deleted file mode 100644 index bcc7c8327..000000000 --- a/src/lighteval/tasks/multilingual/tasks/squad_it.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -name: -Squad It - -dataset: -crux82/squad_it - -abstract: -SQuAD-it: Italian translation of the SQuAD dataset. - -languages: -italian - -tags: -multilingual, qa - -paper: -https://github.com/crux82/squad-it -""" - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"squad_{Language.ITALIAN.value}", - prompt_function=get_qa_prompt_function( - Language.ITALIAN, - lambda line: { - "question": line["question"], - "context": line["context"], - "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0], - }, - ), - hf_repo="crux82/squad_it", - hf_subset="default", - hf_filter=lambda line: any(len(ans) > 0 for ans in line["answers"]["text"]), - evaluation_splits=("test",), - few_shots_split="train", - generation_size=400, - stop_sequence=("\n",), - metrics=( - MultilingualQuasiExactMatchMetric(Language.ITALIAN, "prefix"), - MultilingualQuasiF1ScoreMetric(Language.ITALIAN), - ), - ) -] diff --git a/src/lighteval/tasks/multilingual/tasks/swahili_arc.py b/src/lighteval/tasks/multilingual/tasks/swahili_arc.py deleted file mode 100644 index 9cf07ef85..000000000 --- a/src/lighteval/tasks/multilingual/tasks/swahili_arc.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -name: -Swahili Arc - -dataset: - -abstract: -Swahili Arc multilingual benchmark. - -languages: -swahili - -tags: -multilingual, multiple-choice, reasoning - -paper: -""" - -from string import ascii_uppercase - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbPMINorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"community_arc_{Language.SWAHILI.value}_{formulation.name.lower()}:{subset}", - prompt_function=get_mcq_prompt_function( - Language.SWAHILI, - lambda line: { - "question": line["question"], - "choices": line["choices"]["text"], - "gold_idx": int(line["answerKey"]) - 1 - if line["answerKey"].isdigit() - else ascii_uppercase.index(line["answerKey"]), - }, - formulation=formulation, - ), - hf_repo=f"Mollel/ARC_{subset.capitalize()}_SWH", - hf_subset="default", - hf_revision="5347439d3193c8a0dabaab3819914bf076dc94d4" - if subset == "easy" - else "dc1df9df632d14c251594d9129fb833d2ca4429c", - evaluation_splits=("test",), - few_shots_split="train", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ] - + ([LogLikelihoodAccMetric(normalization=LogProbPMINorm())] if subset == "challenge" else []), # type: ignore - ), - ) - for subset in ["easy", "challenge"] - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/thai_exams.py b/src/lighteval/tasks/multilingual/tasks/thai_exams.py deleted file mode 100644 index 79c559e2b..000000000 --- a/src/lighteval/tasks/multilingual/tasks/thai_exams.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -name: -Thai Exams - -dataset: -scb10x/thai_exam - -abstract: -Thai Exams multilingual benchmark. - -languages: -thai - -tags: -knowledge, multilingual, multiple-choice - -paper: -""" - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.adapters import ( - thai_exams_adapter, -) -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -THAI_EXAMS_SUBSETS = ["a_level", "ic", "onet", "tgat", "tpat1"] - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"thai_exams_{Language.THAI.value}_{formulation.name.lower()}:{subset}", - prompt_function=get_mcq_prompt_function(Language.THAI, thai_exams_adapter, formulation=formulation), - hf_repo="scb10x/thai_exam", - hf_subset=subset, - evaluation_splits=("test",), - few_shots_split="train", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for subset in THAI_EXAMS_SUBSETS - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/thaiqa.py b/src/lighteval/tasks/multilingual/tasks/thaiqa.py deleted file mode 100644 index c7a4216fe..000000000 --- a/src/lighteval/tasks/multilingual/tasks/thaiqa.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -name: -Thaiqa - -dataset: -lighteval/thaiqa_squad_fixed - -abstract: -ThaiQA: A question answering dataset for the Thai language. - -languages: -thai - -tags: -multilingual, qa - -paper: -""" - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"thaiqa_{Language.THAI.value}", - prompt_function=get_qa_prompt_function( - Language.THAI, - lambda line: { - "question": line["question"], - "context": line["context"], - "choices": [ans for ans in line["answers"]["answer"] if len(ans) > 0], - }, - ), - hf_repo="lighteval/thaiqa_squad_fixed", - hf_subset="default", - evaluation_splits=("train",), - few_shots_split="validation", - generation_size=400, - stop_sequence=("\n",), - metrics=( - MultilingualQuasiExactMatchMetric(Language.THAI, "prefix"), - MultilingualQuasiF1ScoreMetric(Language.THAI), - ), - ) -] diff --git a/src/lighteval/tasks/multilingual/tasks/tquad_v2.py b/src/lighteval/tasks/multilingual/tasks/tquad_v2.py deleted file mode 100644 index 4d9adcdf8..000000000 --- a/src/lighteval/tasks/multilingual/tasks/tquad_v2.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -name: -Tquad V2 - -dataset: -erdometo/tquad2 - -abstract: -TQuAD v2: Turkish Question Answering Dataset version 2. - -languages: -turkish - -tags: -multilingual, qa - -paper: -""" - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"tquadv2_{Language.TURKISH.value}", - prompt_function=get_qa_prompt_function( - Language.TURKISH, - lambda line: { - "question": line["question"], - "context": line["context"], - "choices": [a["text"] for a in line["answers"]], - }, - ), - hf_repo="erdometo/tquad2", - hf_subset="default", - evaluation_splits=("validation",), - few_shots_split="train", - generation_size=400, - stop_sequence=("\n",), - metrics=( - MultilingualQuasiExactMatchMetric(Language.TURKISH, "prefix"), - MultilingualQuasiF1ScoreMetric(Language.TURKISH), - ), - ) -] diff --git a/src/lighteval/tasks/multilingual/tasks/turkic.py b/src/lighteval/tasks/multilingual/tasks/turkic.py deleted file mode 100644 index b7992535a..000000000 --- a/src/lighteval/tasks/multilingual/tasks/turkic.py +++ /dev/null @@ -1,121 +0,0 @@ -""" -name: -Turkic Evals - -dataset: -jafarisbarov/TUMLU-mini - -abstract: -TUMLU-mini is a benchmark for Turkic language understanding, comprising 1,000 -prompts organized into 10 subsets. - -languages: -turkic - -tags: -knowledge, multiple-choice - -paper: -https://arxiv.org/abs/2502.11020 -""" - -from functools import partial - -from lighteval.metrics.metrics import Metrics -from lighteval.metrics.normalizations import LogProbCharNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -# TUMLU -# fmt: off -TUMLU_SUBSETS = [ - "azerbaijani", - "crimean-tatar", - "karakalpak", - "kazakh", - "tatar", - "turkish", - "uyghur", - "uzbek", - "kyrgyz" -] -# fmt: on - -INSTRUCTION_BY_LANGUAGE = { - "azerbaijani": "Aşağıdakı sual çoxvariantlı sualdır. Düzgün cavabı seçin:\n\n", - "crimean-tatar": "Aşağıdaki sual çoqtan-çoq cevaplı sualdir. Doğru cevapnı seçip alıñız:\n\n", - "karakalpak": "Tómendegi soraw kóp tańlawlı soraw Tuwrı juwaptı saylań:\n\n", - "kazakh": "Төмендегі сұрақ көп таңдау мүмкіндігі бар сұрақ. Дұрыс жауапты таңдаңыз:\n\n", - "tatar": "Түбәндәге сорау - күп сорау. Дөрес җавапны сайлагыз:\n\n", - "turkish": "Aşağıdaki soru çoktan seçmeli bir sorudur. Doğru cevabı seçin:\n\n", - "uyghur": "تۆۋەندىكى سوئال كۆپ تاللاش سوئالى. توغرا جاۋابنى تاللاڭ:\n\n", - "uzbek": "Quyidagi savol tanlovli savoldir. To‘g‘ri javobni tanlang:\n\n", - "kyrgyz": "Төмөнкү суроо бир нече варианттуу суроо. Туура жоопту тандаңыз:\n\n", -} - -ANSWER_BY_LANGUAGE = { - "uzbek": "Javob:", - "uzbek-cyrillic": "Жавоб", - "crimean-tatar": "Cevap:", - "crimean-tatar-cyrillic": "Джевап", - "tatar": "Җавап:", - "kazakh": "Жауап:", - "kazakh-latin": "Jawap", - "karakalpak": "Juwap:", - "kyrgyz": "Жооп:", - "turkish": "Cevap:", - "uyghur": "جاۋاب:", - "uyghur-latin": "Jawab:", - "azerbaijani": "Cavab:", -} - - -def tumlu_pfn(line, task_name: str = None, language: str = None): - instruction = INSTRUCTION_BY_LANGUAGE[language] - - # Create a list of valid choices with corresponding keys - choices = line.get("choices") - valid_keys = ["A", "B", "C", "D", "E"][: len(choices)] - - answer_index = valid_keys.index(line.get("answer")) - - # Construct the query - query = f"{instruction}{line['question']}\n" - query += "".join([f"{key}. {choice}\n" for key, choice in zip(valid_keys, choices)]) - query += ANSWER_BY_LANGUAGE[language] - - return Doc( - task_name=task_name, - query=query, - choices=valid_keys, # Return only valid choices - gold_index=answer_index, # Correct index - instruction=instruction, - ) - - -class CustomTUMLUTask(LightevalTaskConfig): - def __init__( - self, - name, - hf_subset, - ): - super().__init__( - name=name, - hf_subset=hf_subset, - prompt_function=partial(tumlu_pfn, language=hf_subset), - hf_repo="jafarisbarov/TUMLU-mini", - metrics=[Metrics.loglikelihood_acc(sample_params={"logprob_normalization": LogProbCharNorm()})], - hf_avail_splits=["test", "dev"], - evaluation_splits=["test"], - few_shots_split=["dev"], - few_shots_select="sequential", - generation_size=-1, - stop_sequence=None, - version=0, - ) - - -TUMLU_TASKS = [CustomTUMLUTask(name=f"tumlu:{subset}", hf_subset=subset) for subset in TUMLU_SUBSETS] - -TASKS_TABLE = TUMLU_TASKS diff --git a/src/lighteval/tasks/multilingual/tasks/turkish_arc.py b/src/lighteval/tasks/multilingual/tasks/turkish_arc.py deleted file mode 100644 index 22183d352..000000000 --- a/src/lighteval/tasks/multilingual/tasks/turkish_arc.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -name: -Turkish Arc - -dataset: -malhajar/arc-tr - -abstract: -Turkish ARC Comes from the Turkish leaderboard - -languages: -turkish - -tags: -multilingual, multiple-choice, reasoning - -paper: -""" - -from string import ascii_uppercase - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbPMINorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"community_arc_{Language.TURKISH.value}_{formulation.name.lower()}:{subset}", - prompt_function=get_mcq_prompt_function( - Language.TURKISH, - lambda line: { - "question": line["question"], - "choices": line["choices"]["text"], - "gold_idx": int(line["answerKey"]) - 1 - if line["answerKey"].isdigit() - else ascii_uppercase.index(line["answerKey"]), - }, - formulation=formulation, - ), - hf_repo="malhajar/arc-tr", - hf_subset=f"ARC-{subset.capitalize()}", - evaluation_splits=("test",), - hf_avail_splits=["train"], - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ] - + ([LogLikelihoodAccMetric(normalization=LogProbPMINorm())] if subset == "challenge" else []), # type: ignore - ), - ) - for subset in ["easy", "challenge"] - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/turkish_mmlu.py b/src/lighteval/tasks/multilingual/tasks/turkish_mmlu.py deleted file mode 100644 index 410268f9e..000000000 --- a/src/lighteval/tasks/multilingual/tasks/turkish_mmlu.py +++ /dev/null @@ -1,81 +0,0 @@ -""" -name: -Turkish Mmlu - -dataset: -AYueksel/TurkishMMLU - -abstract: -Turkish Mmlu multilingual benchmark. - -languages: -turkish - -tags: -knowledge, multilingual, multiple-choice - -paper: -""" - -from string import ascii_uppercase - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbPMINorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation, normalize_subset -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TURKISH_MMLU_SUBSET = [ - "Biology", - "Chemistry", - "Geography", - "History", - "Mathematics", - "Philosophy", - "Physics", - "Religion_and_Ethics", - "Turkish_Language_and_Literature", -] - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"mmlu_{Language.TURKISH.value}_{formulation.name.lower()}:{normalize_subset(subset)}", - prompt_function=get_mcq_prompt_function( - Language.TURKISH, - lambda line: { - "question": line["question"], - "choices": line["choices"], - "gold_idx": ascii_uppercase.index(line["answer"]), - }, - formulation=formulation, - ), - hf_repo="AYueksel/TurkishMMLU", - hf_subset=subset, - evaluation_splits=("test",), - few_shots_split="dev", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbPMINorm()), - ], - ), - ) - for subset in TURKISH_MMLU_SUBSET - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/tydiqa.py b/src/lighteval/tasks/multilingual/tasks/tydiqa.py deleted file mode 100644 index 241707a32..000000000 --- a/src/lighteval/tasks/multilingual/tasks/tydiqa.py +++ /dev/null @@ -1,65 +0,0 @@ -""" -name: -Tydiqa - -dataset: -google-research-datasets/tydiqa - -abstract: -Other QA tasks for RC TyDi QA: A benchmark for information-seeking question answering in typologically diverse languages. https://arxiv.org/abs/2003.05002 - -languages: -arabic, bengali, english, finnish, indonesian, japanese, korean, russian, swahili, telugu, thai - -tags: -multilingual, qa - -paper: -https://arxiv.org/abs/2003.05002 -""" - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"tydiqa_{language.value}", - prompt_function=get_qa_prompt_function( - language, - lambda line: { - "question": line["question"], - "context": line["context"], - "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0], - }, - ), - hf_repo="google-research-datasets/tydiqa", - hf_subset="secondary_task", - evaluation_splits=("validation",), - few_shots_split="train", - generation_size=400, - stop_sequence=("\n",), - metrics=( - MultilingualQuasiExactMatchMetric(language, "prefix"), - MultilingualQuasiF1ScoreMetric(language), - ), - ) - for language in [ - Language.ENGLISH, - Language.ARABIC, - Language.BENGALI, - Language.FINNISH, - Language.INDONESIAN, - Language.JAPANESE, - Language.KOREAN, - Language.SWAHILI, - Language.RUSSIAN, - Language.TELUGU, - Language.THAI, - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/worldtree_rus.py b/src/lighteval/tasks/multilingual/tasks/worldtree_rus.py deleted file mode 100644 index 237a90b3d..000000000 --- a/src/lighteval/tasks/multilingual/tasks/worldtree_rus.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -name: -Worldtree Rus - -dataset: -ai-forever/MERA - -abstract: -WorldTree is a dataset for multi-hop inference in science question answering. It -provides explanations for elementary science questions by combining facts from a -semi-structured knowledge base. This Russian version is part of the MERA -(Multilingual Evaluation of Reasoning Abilities) benchmark. - -languages: -russian - -tags: -multilingual - -paper: -https://github.com/ai-forever/MERA -""" - -from string import ascii_uppercase - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"mera_worldtree_{Language.RUSSIAN.value}_{formulation.name.lower()}", - prompt_function=get_mcq_prompt_function( - Language.RUSSIAN, - lambda line: { - "question": line["inputs"]["question"], - "choices": [line["inputs"][f"option_{i.lower()}"] for i in ascii_uppercase[:4]], - "gold_idx": ascii_uppercase.index(line["outputs"]), - }, - formulation=formulation, - ), - hf_repo="ai-forever/MERA", - hf_subset="ruworldtree", - evaluation_splits=("train",), - hf_avail_splits=["train"], - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/xcodah.py b/src/lighteval/tasks/multilingual/tasks/xcodah.py deleted file mode 100644 index 1a7028aeb..000000000 --- a/src/lighteval/tasks/multilingual/tasks/xcodah.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -name: -Xcodah - -dataset: -INK-USC/xcsr - -abstract: -Xcodah multilingual benchmark. - -languages: -arabic, chinese, dutch, english, french, german, hindi, italian, japanese, -polish, portuguese, russian, spanish, swahili, urdu, vietnamese - -tags: -multilingual, multiple-choice, reasoning - -paper: -""" - -from functools import partial - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.adapters import ( - xcodah_adapter, -) -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"xcodah_{language.value}_{formulation.name.lower()}", - prompt_function=get_mcq_prompt_function(language, partial(xcodah_adapter, language), formulation=formulation), - hf_repo="INK-USC/xcsr", - hf_subset=f"X-CODAH-{standardize_tag(language.value) if language != Language.JAPANESE else 'jap'}", - evaluation_splits=("validation",), - hf_avail_splits=["validation"], - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for language in [ - Language.ARABIC, - Language.GERMAN, - Language.ENGLISH, - Language.SPANISH, - Language.FRENCH, - Language.HINDI, - Language.ITALIAN, - Language.JAPANESE, - Language.DUTCH, - Language.POLISH, - Language.PORTUGUESE, - Language.RUSSIAN, - Language.SWAHILI, - Language.URDU, - Language.VIETNAMESE, - Language.CHINESE, - ] - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/xcopa.py b/src/lighteval/tasks/multilingual/tasks/xcopa.py deleted file mode 100644 index 4251126f0..000000000 --- a/src/lighteval/tasks/multilingual/tasks/xcopa.py +++ /dev/null @@ -1,81 +0,0 @@ -""" -name: -Xcopa - -dataset: - -abstract: -COPA (Choice of Plausible Alternatives) tasks involve determining the most -plausible cause or effect for a given premise. These tasks test common sense -reasoning and causal inference abilities. XCOPA: Cross-lingual Choice of -Plausible Alternatives. - -languages: -arabic, chinese, estonian, haitian, indonesian, italian, quechua, swahili, -tamil, thai, turkish, vietnamese - -tags: -multilingual, multiple-choice, narrative, reasoning - -paper: -https://aclanthology.org/2020.emnlp-main.185/ -""" - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.copa import get_copa_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"xcopa_{language.value}_{formulation.name.lower()}", - prompt_function=get_copa_prompt_function( - language, - adapter=lambda line: { - "context": line["premise"], - "cause_effect": line["question"], - "continuations": [line["choice1"], line["choice2"]], - "gold_idx": int(line["label"]), - }, - formulation=formulation, - ), - hf_repo=("OALL/AlGhafa-Arabic-LLM-Benchmark-Translated" if language == Language.ARABIC else "xcopa"), - hf_subset=("copa_ext_ar" if language == Language.ARABIC else standardize_tag(language.value)), - evaluation_splits=["test"], - few_shots_split="validation", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for language in [ - Language.ARABIC, - Language.ESTONIAN, - Language.INDONESIAN, - Language.ITALIAN, - Language.SWAHILI, - Language.TAMIL, - Language.THAI, - Language.TURKISH, - Language.VIETNAMESE, - Language.CHINESE, - Language.HAITIAN, - Language.QUECHUA, - ] - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] -] diff --git a/src/lighteval/tasks/multilingual/tasks/xcsqa.py b/src/lighteval/tasks/multilingual/tasks/xcsqa.py deleted file mode 100644 index 984898990..000000000 --- a/src/lighteval/tasks/multilingual/tasks/xcsqa.py +++ /dev/null @@ -1,94 +0,0 @@ -""" -name: -Xcsqa - -dataset: -INK-USC/xcsr - -abstract: -XCSQA (Cross-lingual Commonsense QA) is part of the XCSR (Cross-lingual -Commonsense Reasoning) benchmark It is a multilingual extension of the -CommonsenseQA dataset, covering 16 languages The task involves answering -multiple-choice questions that require commonsense reasoning Uses PMI -normalization. - -languages: -arabic, chinese, dutch, english, french, german, hindi, italian, japanese, -polish, portuguese, russian, spanish, swahili, urdu, vietnamese - -tags: -multilingual, multiple-choice, qa, reasoning - -paper: -https://arxiv.org/abs/2110.08462 -""" - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbPMINorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"xcsqa_{language.value}_{formulation.name.lower()}", - prompt_function=get_mcq_prompt_function( - language, - lambda line: { - "question": line["question"]["stem"], - "choices": line["question"]["choices"]["text"], - "gold_idx": line["question"]["choices"]["label"].index(line["answerKey"]), - }, - formulation=formulation, - ), - hf_repo="INK-USC/xcsr", - hf_subset=f"X-CSQA-{standardize_tag(language.value) if language != Language.JAPANESE else 'jap'}", - hf_filter=lambda x: all( - len(x["question"]["choices"]["text"][i].strip()) > 0 for i in range(len(x["question"]["choices"]["text"])) - ), - evaluation_splits=("validation",), - hf_avail_splits=["validation"], - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - LogLikelihoodAccMetric(normalization=LogProbPMINorm()), - ], - ), - ) - for language in [ - Language.ARABIC, - Language.GERMAN, - Language.ENGLISH, - Language.SPANISH, - Language.FRENCH, - Language.HINDI, - Language.ITALIAN, - Language.JAPANESE, - Language.DUTCH, - Language.POLISH, - Language.PORTUGUESE, - Language.RUSSIAN, - Language.SWAHILI, - Language.URDU, - Language.VIETNAMESE, - Language.CHINESE, - ] - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/xnli.py b/src/lighteval/tasks/multilingual/tasks/xnli.py deleted file mode 100644 index 08ddde2f2..000000000 --- a/src/lighteval/tasks/multilingual/tasks/xnli.py +++ /dev/null @@ -1,92 +0,0 @@ -""" -name: -Xnli - -dataset: -facebook/xnli - -abstract: -NLI (Natural Language Inference) tasks involve determining the logical -relationship between two given sentences: a premise and a hypothesis. The goal -is to classify whether the hypothesis is entailed by, contradicts, or is neutral -with respect to the premise. After our inspection we found the neutral label to -be quite ambiguous and decided to exclude it. But you can easily add it by -modifying the adapters The XNLI dataset is a multilingual variant of MultiNLI - -languages: -arabic, bulgarian, chinese, english, french, german, greek, hindi, russian, -spanish, swahili, thai, turkish, urdu, vietnamese - -tags: -classification, multilingual, nli - -paper: -https://aclanthology.org/D18-1269/ -""" - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.nli import get_nli_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"xnli_{language.value}_{formulation.name.lower()}", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=None), - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - prompt_function=get_nli_prompt_function( - language=language, - adapter=lambda line: { - "premise": line["premise"], - "hypothesis": line["hypothesis"], - # Since we ignore the neutral label - "gold_idx": {0: 0, 2: 1}[line["label"]], - }, - relations=["entailment", "contradiction"], - formulation=formulation, - ), - hf_filter=lambda line: line["label"] in [0, 2], - hf_repo="facebook/xnli", - hf_subset=standardize_tag(language.value), - evaluation_splits=["validation"], - few_shots_split="train", - ) - for language in [ - Language.ARABIC, - Language.ENGLISH, - Language.FRENCH, - Language.SPANISH, - Language.BULGARIAN, - Language.GERMAN, - Language.GREEK, - Language.ENGLISH, - Language.FRENCH, - Language.HINDI, - Language.RUSSIAN, - Language.SWAHILI, - Language.THAI, - Language.TURKISH, - Language.URDU, - Language.VIETNAMESE, - Language.CHINESE, - ] - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] -] diff --git a/src/lighteval/tasks/multilingual/tasks/xnli2.py b/src/lighteval/tasks/multilingual/tasks/xnli2.py deleted file mode 100644 index 59f4facaf..000000000 --- a/src/lighteval/tasks/multilingual/tasks/xnli2.py +++ /dev/null @@ -1,99 +0,0 @@ -""" -name: -Xnli2 - -dataset: - -abstract: -Improvement on XNLI with better translation, from our experience models tend to -perform better on XNLI2.0 than XNLI. - -languages: -arabic, assamese, bengali, bulgarian, chinese, english, french, german, greek, -gujarati, hindi, kannada, marathi, punjabi, russian, sanskrit, spanish, swahili, -tamil, thai, turkish, urdu, vietnamese - -tags: -classification, multilingual, nli - -paper: -https://arxiv.org/abs/2301.06527 -""" - -from langcodes import Language as LangCodeLanguage -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.nli import get_nli_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"xnli2.0_{language.value}_{formulation.name.lower()}", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=None), - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - prompt_function=get_nli_prompt_function( - language=language, - adapter=lambda line: { - "premise": line["premise"], - "hypothesis": line["hypothesis"], - # Since we ignore the neutral label - "gold_idx": {0: 0, 2: 1}[line["label"]], - }, - relations=["entailment", "contradiction"], - formulation=formulation, - ), - hf_filter=lambda line: line["label"] in [0, 2] - and line["premise"] is not None - and line["hypothesis"] is not None, - hf_repo=f"Harsit/xnli2.0_train_{LangCodeLanguage(standardize_tag(language.value)).language_name().lower()}", - hf_subset="default", - evaluation_splits=["train"], - hf_avail_splits=["train"], - ) - for language in [ - Language.ENGLISH, - Language.FRENCH, - Language.PUNJABI, - Language.GUJARATI, - Language.KANNADA, - Language.ASSAMESE, - Language.BENGALI, - Language.MARATHI, - Language.SANSKRIT, - Language.TAMIL, - Language.GERMAN, - Language.ENGLISH, - Language.URDU, - Language.VIETNAMESE, - Language.TURKISH, - Language.THAI, - Language.SWAHILI, - Language.SPANISH, - Language.RUSSIAN, - Language.HINDI, - Language.GREEK, - Language.CHINESE, - Language.BULGARIAN, - Language.ARABIC, - # Theoretically also: Bhojpuri, Gujarati, Odiya - ] - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] -] diff --git a/src/lighteval/tasks/multilingual/tasks/xnli_indic.py b/src/lighteval/tasks/multilingual/tasks/xnli_indic.py deleted file mode 100644 index 7c77af516..000000000 --- a/src/lighteval/tasks/multilingual/tasks/xnli_indic.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -name: -Xnli Indic - -dataset: -Divyanshu/indicxnli - -abstract: -Another variant of XNLI, with emphasis on Indic languages. - -languages: -assamese, bengali, gujarati, hindi, kannada, malayalam, marathi, oriya, punjabi, -tamil, telugu - -tags: -classification, multilingual, nli - -paper: -https://arxiv.org/abs/2204.08776 -""" - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.nli import get_nli_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"indicnxnli_{language.value}_{formulation.name.lower()}", - prompt_function=get_nli_prompt_function( - language=language, - adapter=lambda line: { - "premise": line["premise"], - "hypothesis": line["hypothesis"], - # Since we ignore the neutral label - "gold_idx": {0: 0, 2: 1}[line["label"]], - }, - relations=["entailment", "contradiction"], - formulation=formulation, - ), - hf_repo="Divyanshu/indicxnli", - hf_subset=standardize_tag(language.value), - # Ignore neutral - hf_filter=lambda x: int(x["label"]) in [0, 2], - evaluation_splits=["validation"], - few_shots_split="train", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=None), - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for language in [ - Language.ASSAMESE, - Language.BENGALI, - Language.GUJARATI, - Language.HINDI, - Language.KANNADA, - Language.MALAYALAM, - Language.MARATHI, - Language.ORIYA, - Language.PUNJABI, - Language.TAMIL, - Language.TELUGU, - ] - for formulation in [MCFFormulation(), CFFormulation(), HybridFormulation()] -] diff --git a/src/lighteval/tasks/multilingual/tasks/xquad.py b/src/lighteval/tasks/multilingual/tasks/xquad.py deleted file mode 100644 index e2c306f4d..000000000 --- a/src/lighteval/tasks/multilingual/tasks/xquad.py +++ /dev/null @@ -1,73 +0,0 @@ -""" -name: -Xquad - -dataset: -google/xquad - -abstract: -Reading Comprehension (RC) tasks evaluate a model's ability to understand and -extract information from text passages. These tasks typically involve answering -questions based on given contexts, spanning multiple languages and formats. Add -RC tasks supporting about 130 unique languages/scripts. SQuAD - like XQuAD: -Cross-lingual Question Answering Dataset, extending SQuAD to 11 languages. - -languages: -arabic, chinese, english, german, greek, hindi, romanian, russian, spanish, -thai, turkish, vietnamese - -tags: -multilingual, qa - -paper: -https://arxiv.org/abs/1910.11856 -""" - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"xquad_{language.value}", - prompt_function=get_qa_prompt_function( - language, - lambda line: { - "question": line["question"], - "context": line["context"], - "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0], - }, - ), - hf_repo="google/xquad", - hf_subset=f"xquad.{standardize_tag(language.value)}", - evaluation_splits=("validation",), - few_shots_split="validation", - generation_size=400, - stop_sequence=("\n",), - metrics=( - MultilingualQuasiExactMatchMetric(language, "prefix"), - MultilingualQuasiF1ScoreMetric(language), - ), - ) - for language in [ - Language.ARABIC, - Language.GERMAN, - Language.GREEK, - Language.ENGLISH, - Language.SPANISH, - Language.HINDI, - Language.ROMANIAN, - Language.RUSSIAN, - Language.THAI, - Language.TURKISH, - Language.VIETNAMESE, - Language.CHINESE, - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/xstory.py b/src/lighteval/tasks/multilingual/tasks/xstory.py deleted file mode 100644 index d1d450cdb..000000000 --- a/src/lighteval/tasks/multilingual/tasks/xstory.py +++ /dev/null @@ -1,92 +0,0 @@ -""" -name: -Xstory - -dataset: -juletxara/xstory_cloze - -abstract: -Xstory multilingual benchmark. - -languages: -arabic, basque, burmese, chinese, hindi, indonesian, russian, spanish, swahili, -telugu - -tags: -multilingual, narrative - -paper: -""" - -from functools import partial - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.utils.task_utils import get_metrics_for_formulation -from lighteval.tasks.templates.continuation import get_continuation_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.tasks.templates.utils.translation_literals import TRANSLATION_LITERALS -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"xstory_cloze_{lang.value}_{formulation.name.lower()}", - prompt_function=get_continuation_prompt_function( - lang, - partial( - lambda lang, line: { - "context": TRANSLATION_LITERALS[lang].sentence_space.join( - [ - line["input_sentence_1"], - line["input_sentence_2"], - line["input_sentence_3"], - line["input_sentence_4"], - ] - ), - "continuations": [line["sentence_quiz1"], line["sentence_quiz2"]], - "gold_idx": int(line["answer_right_ending"]) - 1, # type: ignore - }, - lang, - ), - formulation=formulation, - ), - hf_repo="juletxara/xstory_cloze", - hf_subset=standardize_tag(lang.value), - evaluation_splits=["eval"], - few_shots_split="train", - metrics=get_metrics_for_formulation( - formulation, - [ - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ), - ) - for lang in [ - Language.RUSSIAN, - Language.CHINESE, - Language.SPANISH, - Language.ARABIC, - Language.HINDI, - Language.INDONESIAN, - Language.TELUGU, - Language.SWAHILI, - Language.BASQUE, - Language.BURMESE, - ] - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/tasks/xwinograd.py b/src/lighteval/tasks/multilingual/tasks/xwinograd.py deleted file mode 100644 index b83240afa..000000000 --- a/src/lighteval/tasks/multilingual/tasks/xwinograd.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -name: -Xwinograd - -dataset: -Muennighoff/xwinograd - -abstract: -Xwinograd multilingual benchmark. - -languages: -chinese, english, french, japanese, portuguese, russian - -tags: -multilingual, multiple-choice, reasoning - -paper: -""" - -from functools import partial - -from langcodes import standardize_tag - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, -) -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbTokenNorm -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.multilingual.adapters import ( - winogrand_adapter, -) -from lighteval.tasks.templates.continuation import get_continuation_prompt_function -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - HybridFormulation, - MCFFormulation, -) -from lighteval.utils.language import Language - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"xwinograd_{language.value}_{formulation.name.lower()}", - prompt_function=get_continuation_prompt_function( - language, partial(winogrand_adapter, language), formulation=formulation - ), - hf_repo="Muennighoff/xwinograd", - hf_subset=standardize_tag(language.value) if language != Language.JAPANESE else "jp", - evaluation_splits=("test",), - hf_avail_splits=["test"], - metrics=[ - LogLikelihoodAccMetric(normalization=None), - LogLikelihoodAccMetric(normalization=LogProbTokenNorm()), - LogLikelihoodAccMetric(normalization=LogProbCharNorm()), - ], - ) - for language in [ - Language.ENGLISH, - Language.FRENCH, - Language.JAPANESE, - Language.PORTUGUESE, - Language.RUSSIAN, - Language.CHINESE, - ] - for formulation in [ - MCFFormulation(), - CFFormulation(), - HybridFormulation(), - ] -] diff --git a/src/lighteval/tasks/multilingual/utils/__init__.py b/src/lighteval/tasks/multilingual/utils/__init__.py deleted file mode 100644 index a732db8d0..000000000 --- a/src/lighteval/tasks/multilingual/utils/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. diff --git a/src/lighteval/tasks/multilingual/utils/adapters_utils.py b/src/lighteval/tasks/multilingual/utils/adapters_utils.py deleted file mode 100644 index b82e9c434..000000000 --- a/src/lighteval/tasks/multilingual/utils/adapters_utils.py +++ /dev/null @@ -1,139 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from functools import partial, reduce -from typing import Literal - -from lighteval.tasks.templates.utils.formatting_utils import PUNCT -from lighteval.tasks.templates.utils.translation_literals import TranslationLiterals - - -MULTICHOICE_JOIN_VARIANT = Literal["COMMA", "NEW_LINE"] - - -def multichoice_join(choices: list[str], variant: MULTICHOICE_JOIN_VARIANT, translation_literals: TranslationLiterals): - """Joins the choices with the appropriate separator. - - Returns: - str: The joined choices string with appropriate separators - """ - separator: str - if variant == "COMMA": - separator = f"{translation_literals.comma}{translation_literals.word_space}" - elif variant == "NEW_LINE": - separator = "\n" - - return separator.join(choices) - - -def multichoice_to_single_choice( - choices: list[str], - gold_idx: list[int], - join_variant: MULTICHOICE_JOIN_VARIANT, - translation_literals: TranslationLiterals, -): - """Converts from multi-choice format to single-choice format, by joining the correct choices with the appropriate separator. - - Args: - choices (list[str]): List of choices. - gold_idx (list[int]): List of indices of the correct choices. - join_variant (MULTICHOICE_JOIN_VARIANT): Variant of the separator to join the choices. - translation_literals (TranslationLiterals): Translation literals. - - Returns: - tuple[list[str], list[int]]: Tuple of the new choices and the new gold index. - """ - if len(gold_idx) == 1: - return choices, gold_idx - - multichoice_joiner = partial(multichoice_join, variant=join_variant, translation_literals=translation_literals) - - import random - - correct_choice = multichoice_joiner([choices[i] for i in gold_idx]) - incorrect_choices = [choice for i, choice in enumerate(choices) if i not in gold_idx] - - new_choices = incorrect_choices + [correct_choice] - random.shuffle(new_choices) - - new_gold_idx = new_choices.index(correct_choice) - - return new_choices, [new_gold_idx] - - -def extract_answers_from_string(answer_string: str, answer_prefixes: list[str]) -> tuple[int, dict[str, str]] | None: - """Attempts to extract answers from the answer_string. The answers are identified by being prefixed with answer prefixes. - The extraction is done from the end to the beginning and all answer prefixes must be found in the answer_string. - - Example: - This is a question. ① Yes ② No ③ Yes ④ No - - Expected output: - [ - 21, - {"①": "Yes", "②": "No", "③": "Yes", "④": "No"}, - ] - - Args: - answer_string (str): String possibly containing answers. - answer_prefixes (list[str]): List of answer prefixes. - - Returns: - Optional[tuple[int, dict[str, str]]]: A tuple containing the start index of the answer and dictionary mapping the prefix to the answer. - """ - - def extract_answer(acc: tuple[str, int, list[str]], symbol: str) -> tuple[str, int, list[str]]: - """Extracts an answer from the text until the next symbol is found. - - Args: - acc (tuple[str, int, list[str]]): Tuple containing the text, the right index (where to start searching) and the list of found answers. - symbol (str): Symbol to extract the answer from. - - Returns: - tuple[str, int, list[str]]: Tuple containing the text, the right index and the list of answers. - """ - text, right_index, answers = acc - if right_index == -1: - return text, right_index, answers - left_index = text.rfind(symbol, 0, right_index) - if left_index == -1: - return text, -1, answers - return text, left_index, answers + [text[left_index:right_index]] - - # Try to extract answers from the possible_answers_part - - sorted_answer_prefixes = sorted(answer_prefixes, reverse=True) - _, last_index, found_answers = reduce( - extract_answer, sorted_answer_prefixes, (answer_string, len(answer_string), []) - ) - if last_index == -1: - return None - - # Ensure we have answers for all prefixes - if len(found_answers) != len(answer_prefixes): - return None - - found_answers = [answer.rstrip(PUNCT + ";").strip() for answer in found_answers] - prefix_answer_dict = { - answer[: len(prefix)]: answer[len(prefix) :].strip() for answer, prefix in zip(found_answers, answer_prefixes) - } - return last_index, prefix_answer_dict diff --git a/src/lighteval/tasks/multilingual/utils/task_utils.py b/src/lighteval/tasks/multilingual/utils/task_utils.py deleted file mode 100644 index 5a116f6a3..000000000 --- a/src/lighteval/tasks/multilingual/utils/task_utils.py +++ /dev/null @@ -1,40 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -from lighteval.metrics.dynamic_metrics import LogLikelihoodAccMetric -from lighteval.metrics.utils.metric_utils import Metric -from lighteval.tasks.templates.utils.formulation import Formulation, MCFFormulation - - -def normalize_subset(subset: str) -> str: - return subset.replace(" ", "_").replace("(", "").replace(")", "").lower() - - -def get_metrics_for_formulation(formulation: Formulation, metrics: list[Metric]) -> list[Metric]: - """Choose the appropriate metrics for the given formulation otherwise fallback to the original metrics.""" - match formulation: - # - case MCFFormulation(choice_prefix="Letters"): - return [LogLikelihoodAccMetric(normalization=None)] - case _: - return metrics diff --git a/src/lighteval/tasks/prompt_manager.py b/src/lighteval/tasks/prompt_manager.py deleted file mode 100644 index f72b8050c..000000000 --- a/src/lighteval/tasks/prompt_manager.py +++ /dev/null @@ -1,335 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import logging -import random -from collections import defaultdict -from dataclasses import dataclass -from enum import Enum -from itertools import cycle -from typing import TYPE_CHECKING - -from lighteval.tasks.requests import Doc -from lighteval.utils.utils import as_list - - -logger = logging.getLogger(__name__) - - -if TYPE_CHECKING: - from lighteval.tasks.lighteval_task import LightevalTask - - -class PromptManager: - def __init__(self, use_chat_template: bool = False, tokenizer=None, system_prompt: str | None = None): - self.use_chat_template = use_chat_template - self.tokenizer = tokenizer - self.system_prompt = system_prompt # System prompt to be used in chat templates - - def prepare_prompt(self, doc: Doc) -> str: - """Prepare a prompt from a document, either using chat template or plain text format. - - Returns: - str: The formatted prompt string - """ - if self.use_chat_template: - return self._prepare_chat_template(doc) - else: - return self._prepare_plain_text(doc) - - def prepare_prompt_multimodal(self, doc: Doc) -> str: - if self.use_chat_template is False or self.tokenizer is None: - raise ValueError("Multimodal prompts are only supported with chat template format.") - - if doc.images is None: - raise ValueError("Multimodal prompts require images to be provided in the document.") - - text_content = [{"type": "text", "text": doc.query}] - image_content = [{"type": "image", "image": image} for image in doc.images] - message = {"role": "user", "content": text_content + image_content} - - if ( - self.system_prompt is not None or doc.instruction is not None - ): # We add system prompt and instruction jointly if possible - system_prompt = self.system_prompt if self.system_prompt is not None else "" - instruction = doc.instruction if doc.instruction is not None else "" - system_content = [{"type": "text", "text": system_prompt + instruction}] - system_prompt_message = {"role": "system", "content": system_content} - message = [system_prompt_message, message] - - else: - message = [message] - - return self.tokenizer.apply_chat_template( - message, - tokenize=False, - add_generation_prompt=True, - ) - - def prepare_prompt_api(self, doc: Doc) -> list[dict[str, str]]: - """Prepare a prompt for API calls, using a chat-like format. - Will not tokenize the message because APIs will usually handle this. - - Returns: - list[dict[str, str]]: List of message dictionaries for API calls - """ - return self._prepare_chat_template(doc, tokenize=False) - - def _prepare_chat_template(self, doc: Doc, tokenize: bool = True) -> str: - """Prepare prompt using chat template format. - - Returns: - str | list[dict[str, str]]: Formatted chat template string or list of messages - """ - messages = [] - instruction_used = False # Flag to check if instruction is used in the first few-shot example - - # Add system prompt if available - if self.system_prompt is not None: - messages.append({"role": "system", "content": self.system_prompt}) - - # Add few-shot examples - for ix, fewshot_sample in enumerate(doc.fewshot_samples): - query = self._extract_query(fewshot_sample.query, fewshot_sample.instruction) - if ix == 0 and doc.instruction is not None: - instruction_used = True - query = doc.instruction + query - - messages.append({"role": "user", "content": query}) - messages.append({"role": "assistant", "content": fewshot_sample.get_golds()[0]}) - - # Add main query - main_query = self._extract_query(doc.query, doc.instruction) - - if doc.instruction is not None and not instruction_used: - # If instruction is provided, prepend it to the main query - main_query = doc.instruction + main_query - - messages.append({"role": "user", "content": main_query}) - - if tokenize: # for local models - assert self.tokenizer is not None, "Tokenizer must be set for chat template formatting." - - return self.tokenizer.apply_chat_template( - messages, - tokenize=False, - add_generation_prompt=True, - ) - - else: # for apis - return messages - - def _prepare_plain_text(self, doc: Doc) -> str: - """Prepare prompt using plain text format. - - Returns: - str: The formatted plain text prompt - """ - parts = [] - - # Add system prompt if available - if self.system_prompt is not None: - parts.append(self.system_prompt) - - if doc.instruction is not None: - parts.append(doc.instruction) - - # Add few-shot examples - for fewshot_sample in doc.fewshot_samples: - query = self._extract_query(fewshot_sample.query, fewshot_sample.instruction) - parts.append(query + " " + fewshot_sample.get_golds()[0].strip()) - - # Add main query - query = self._extract_query(doc.query, doc.instruction) - parts.append(query) - - return "\n\n".join(parts) - - def _extract_query(self, query: str, instruction: str | None) -> str: - """Extract query content, removing instruction prefix if appropriate. - - Returns: - str: The extracted query content without instruction prefix if it was present - """ - if instruction is not None: - if query.startswith(instruction): - return query[len(instruction) :].strip() - else: - return query - return query - - -@dataclass -class FewShotSelectionMethod: - sorting: str # sorting method for the overall few shot pool (balanced, random, sequential) - with_sampling: bool # samples item randomly from the few shot pool - fewshotpool_unique: bool # set to true if you are CERTAIN there is no intersection between the few shot pool and your evaluation set - - -class FewShotSelection(Enum): - balanced = FewShotSelectionMethod(sorting="balanced", with_sampling=False, fewshotpool_unique=False) - random = FewShotSelectionMethod(sorting="random", with_sampling=False, fewshotpool_unique=False) - sequential = FewShotSelectionMethod(sorting="sequential", with_sampling=False, fewshotpool_unique=False) - random_sampling_from_train = FewShotSelectionMethod(sorting="random", with_sampling=True, fewshotpool_unique=True) - random_sampling = FewShotSelectionMethod(sorting="random", with_sampling=True, fewshotpool_unique=False) - - -ALLOWED_SELECTIONS = FewShotSelection._member_names_ - - -class FewShotSampler: - def __init__(self, task: "LightevalTask"): - self.task = task - - few_shots_select = task.fewshot_selection - if few_shots_select is None: - few_shots_select = "balanced" - - if few_shots_select not in ALLOWED_SELECTIONS: - raise ValueError( - f"few_shots_select must be one of {','.join(ALLOWED_SELECTIONS[:-1])} or {ALLOWED_SELECTIONS[-1]}, not {few_shots_select}" - ) - - self.few_shots_select = FewShotSelection[few_shots_select] - self.few_shots_split = task.fewshot_split - - self._fewshot_cache = {} - - def sample_fewshot_examples( - self, - num_fewshot: int, - variance_seed: int, - sampler: random.Random | None = None, - formatted_doc: Doc | None = None, - ) -> list[Doc]: - if num_fewshot == 0: - return [] - - self._init_fewshot_pool(num_fewshot=num_fewshot, variance_seed=variance_seed) - samples = self._sample_from_pool(num_fewshot=num_fewshot, variance_seed=variance_seed, sampler=sampler) - - # get rid of the doc that's the one we're evaluating, if it's in the fewshot - return [x for x in samples if x != formatted_doc][:num_fewshot] - - def _init_fewshot_pool( - self, - num_fewshot: int, - variance_seed: int, - ): - # If there is no cache, we initialize it - if variance_seed not in self._fewshot_cache: - if self.few_shots_select.value.sorting == "sequential": - self._init_fewshot_sampling_sequential(num_fewshot=num_fewshot, variance_seed=variance_seed) - elif self.few_shots_select.value.sorting == "random": - self._init_fewshot_sampling_random(variance_seed=variance_seed) - elif self.few_shots_select.value.sorting == "balanced": - self._init_fewshot_sampling_balanced(num_fewshot=num_fewshot, variance_seed=variance_seed) - else: - raise Exception("No correct few shot strategy selected - but this point should not be reachable.") - - def _sample_from_pool(self, variance_seed: int, num_fewshot: int, sampler: random.Random) -> list: - if self.few_shots_select.value.with_sampling and sampler is not None: - if self.few_shots_select.value.fewshotpool_unique: - # This functionality is here for compatibility with the harness few shot system. - # It assumes (in some cases) that there is no intersection between the few shot pool and the actual - # eval examples, and therefore samples only `num_fewshot` (see Task.fewshot_examples) - return sampler.sample(self._fewshot_cache[variance_seed], num_fewshot) - else: # we don't reach this yet but let's add it for future use cases - return sampler.sample(self._fewshot_cache[variance_seed], num_fewshot + 1) - else: - return self._fewshot_cache[variance_seed] - - def _init_fewshot_sampling_sequential(self, num_fewshot: int, variance_seed: int): - # No balancing of the few-shot examples, we take the first items of the set - # We rotate by num_fewshot * seed (seed >= 0) to be able to have different series of sequential few-shots - fewshotpool = self.task.fewshot_docs() - for _ in range(num_fewshot * variance_seed): - fewshotpool.append(fewshotpool.pop(0)) - self._fewshot_cache[variance_seed] = fewshotpool # Store few shot examples - - def _init_fewshot_sampling_random(self, variance_seed: int): - fewshotpool = list(self.task.fewshot_docs()) - if variance_seed == 0: - self._fewshot_cache[variance_seed] = fewshotpool - else: # we shuffle - rnd = random.Random(variance_seed) - rnd.shuffle(fewshotpool) - self._fewshot_cache[variance_seed] = fewshotpool - - def _init_fewshot_sampling_balanced( - self, - num_fewshot: int, - variance_seed: int, - ): - fewshotpool = self.task.fewshot_docs() - - random.seed(variance_seed) - - # Build up balanced selection based on fewshot_sorting_class - # (or the gold target, if the class is undefined) - label_to_instances = defaultdict(list) - for instance in fewshotpool: - target = instance.fewshot_sorting_class or as_list(instance.get_golds())[0] - label_to_instances[target].append(instance) - - # Sort by counts of class labels - counts_to_labels = defaultdict(list) - for label, instances in sorted(label_to_instances.items()): - counts_to_labels[len(instances)].append(label) - - sorted_labels = [] - # Sort the labels by the number of Instances that belong to them - for count in sorted(counts_to_labels, reverse=True): - labels = counts_to_labels[count] - # Break ties by randomly shuffling labels that have the same number of Instances - random.shuffle(labels) - sorted_labels.extend(labels) - - examples = [] - num_instances_to_sample = min( - len(fewshotpool), num_fewshot + 1 - ) # We add 1 to be able to sample for the test set and remove later the doc we are testing on - labels_iterable = cycle(sorted_labels) - while num_instances_to_sample > 0: - next_label = next(labels_iterable, None) - if not next_label: - break - - instances = label_to_instances[next_label] - # If there are no instances to sample for this particular label, skip it. - if len(instances) == 0: - continue - - # Randomly sample without replacement - examples.append(instances.pop(random.randrange(len(instances)))) - num_instances_to_sample -= 1 - - self._fewshot_cache[variance_seed] = examples # Store few shot examples - - def get_fewshot_seeds(self, few_shot_iterations: int = None) -> list[int]: - """Return a list of seeds for sampling several times the few shots""" - # todo @saylortwift: check which seed for bb - if few_shot_iterations <= 1: - return [0] - seeds = range(few_shot_iterations) - logger.warning(f"Running {self.task.name} with {few_shot_iterations} few-shot iterations.") - return seeds diff --git a/src/lighteval/tasks/registry.py b/src/lighteval/tasks/registry.py deleted file mode 100644 index e7c4e9eb6..000000000 --- a/src/lighteval/tasks/registry.py +++ /dev/null @@ -1,562 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import ast -import collections -import copy -import importlib -import importlib.util -import inspect -import logging -import os -import sys -import time -from dataclasses import asdict -from functools import lru_cache -from itertools import groupby -from pathlib import Path -from types import ModuleType - -from lighteval.tasks.lighteval_task import LightevalTask, LightevalTaskConfig - - -# Import community tasks -AVAILABLE_COMMUNITY_TASKS_MODULES = [] - - -def load_community_tasks(): - """Dynamically load community tasks, handling errors gracefully. - - Returns: - list: List of successfully loaded community task modules - """ - modules = [] - try: - # Community tasks are in the lighteval directory, not under src - community_path = Path(__file__).parent.parent.parent.parent / "community_tasks" - if not community_path.exists(): - return modules - - # Ensure the parent directory is on sys.path so we can import `community_tasks.*` - parent_dir = str(community_path.parent) - if parent_dir not in sys.path: - sys.path.insert(0, parent_dir) - - # List all python files in community_tasks - community_files = [p.stem for p in community_path.glob("*.py") if not p.name.startswith("_")] - - for module_name in community_files: - try: - module = importlib.import_module(f"community_tasks.{module_name}") - if hasattr(module, "TASKS_TABLE"): - modules.append(module) - logger.info(f"Successfully loaded community tasks from {module_name}") - except Exception as e: - logger.warning(f"Failed to load community tasks from {module_name}: {e}") - except Exception as e: - logger.warning(f"Error loading community tasks directory: {e}") - - return modules - - -logger = logging.getLogger(__name__) - -# Helm, Bigbench, Harness are implementations following an evaluation suite setup -# Original follows the original implementation as closely as possible -# Leaderboard are the evaluations we fixed on the open llm leaderboard - you should get similar results -# Community are for community added evaluations -# Extended are for evaluations with custom logic -# Custom is for all the experiments you might want to do! - -# Core suites - always available without extra dependencies -CORE_SUITES = [ - "helm", - "bigbench", - "harness", - "leaderboard", - "lighteval", - "original", - "extended", - "custom", - "test", -] - -# Optional suites - may require extra dependencies -OPTIONAL_SUITES = [ - "community", - "multilingual", -] - -DEFAULT_SUITES = CORE_SUITES + OPTIONAL_SUITES - - -class Registry: - """The Registry class is used to manage the task registry and get task classes.""" - - def __init__( - self, - tasks: str | Path | None = None, - load_multilingual: bool = False, - custom_tasks: str | Path | ModuleType | None = None, - ): - """ - Initialize the Registry class. - Registry is responsible for holding a dict of task and their config, initializing a LightevalTask instance when asked. - - Args: - tasks: Task specification string or path to file containing task list. - custom_tasks: Custom tasks to be included in the registry. Can be: - - A string path to a Python file containing custom tasks - - A Path object pointing to a custom tasks file - - A module object containing custom task configurations - - None for default behavior (no custom tasks) - load_multilingual: Whether to load multilingual tasks. - - Each custom task module should contain a TASKS_TABLE exposing - a list of LightevalTaskConfig objects. - - Example: - TASKS_TABLE = [ - LightevalTaskConfig( - name="custom_task", - suite="custom", - ... - ) - ] - """ - if tasks is None: - logger.warning( - "You passed no task name. This should only occur if you are using the CLI to inspect tasks." - ) - self.tasks_list = [] - else: - self.tasks_list = self._get_full_task_list_from_input_string(tasks) - - self._load_multilingual = load_multilingual - self._custom_tasks = custom_tasks - - self._task_registry = Registry.load_all_task_configs( - custom_tasks=custom_tasks, load_multilingual=load_multilingual - ) - self.task_to_configs = self._update_task_configs() - - def _get_full_task_list_from_input_string(self, tasks: str | Path) -> list[str]: - """Converts an input string (either a path to file with a list of tasks or a string of comma-separated tasks) into an actual list""" - if os.path.exists(tasks): - with open(tasks, "r") as f: - tasks_list = [line.strip() for line in f if line.strip() and not line.startswith("#")] - else: - tasks_list = tasks.split(",") - - task_groups = {} - expanded_tasks_list: list[str] = [] - for maybe_task_group in tasks_list: - # We either expand the group (in case it's a group name), or we keep it as is (in case it's a task name or superset name) - expanded_tasks = task_groups.get(maybe_task_group, [maybe_task_group]) - if len(expanded_tasks) > 1: - logger.info(f"Expanding task group {maybe_task_group} to {expanded_tasks}") - expanded_tasks_list.extend(expanded_tasks) - - # We remove exact duplicates - expanded_tasks_list = list(set(expanded_tasks_list)) - - return expanded_tasks_list - - def _update_task_configs(self) -> dict[str, LightevalTaskConfig]: # noqa: C901 - """ - Updates each config depending on the input tasks (we replace all provided params, like few shot number, sampling params, etc) - Now expects task specs in the form: - - task|few_shot - - task (defaults to few_shot=0) - Backwards-compat for suite|task|few_shot is preserved but the suite is ignored. - """ - task_to_configs = collections.defaultdict(list) - - # We map all tasks to their parameters - for task in self.tasks_list: - metric_params_dict = {} - try: - if task.count("|") == 2: - logger.warning( - "Deprecation warning: Provided suite|task|few_shot; `suite` is ignored. Use task|few_shot instead." - ) - _, task_name, few_shot = tuple(task.split("|")) - elif task.count("|") == 1: - task_name, few_shot = tuple(task.split("|")) - else: - task_name = task - few_shot = 0 - if "@" in task_name: - split_task_name = task_name.split("@") - task_name, metric_params = split_task_name[0], split_task_name[1:] - # We convert k:v to {"k": "v"}, then to correct type - metric_params_dict = dict(item.split("=") for item in metric_params if item) - metric_params_dict = {k: ast.literal_eval(v) for k, v in metric_params_dict.items()} - few_shot = int(few_shot) - - except ValueError: - raise ValueError(f"Cannot get task info from {task}. correct format is task|few_shot") - - # This adds support for task supersets (eg: mmlu -> all the mmlu tasks) - for expanded_task in self._expand_task_definition(task_name): - # todo: it's likely we'll want this step at the list set up step, not here - - # We load each config - config = self._task_registry.get(expanded_task) - if config is None: - raise ValueError(f"Cannot find task {expanded_task} in task list or in custom task registry") - - config = copy.deepcopy(config) - config.num_fewshots = few_shot - config.full_name = f"{expanded_task}|{config.num_fewshots}" - # If some tasks are parametrizable and in cli, we set attributes here - for metric in [m for m in config.metrics if "@" in m.metric_name]: # parametrizable metric - for attribute, value in metric_params_dict.items(): - setattr(metric.sample_level_fn, attribute, value) - required = getattr(metric.sample_level_fn, "attribute_must_be_set", []) - for attribute in required: - if getattr(metric.sample_level_fn, attribute) is None: - raise ValueError( - f"Metric {metric.metric_name} for task {expanded_task} " - f"was not correctly parametrized. Forgot to set '{attribute}'." - ) - - task_to_configs[expanded_task].append(config) - - return task_to_configs - - def load_tasks(self) -> dict[str, LightevalTask]: - if len(self.task_to_configs) == 0: # we're in cli to analyse tasks, we return all tasks - return {f"{config.full_name}": LightevalTask(config=config) for config in self._task_registry.values()} - - # We return only the tasks of interest - return { - f"{config.full_name}": LightevalTask(config=config) - for configs in self.task_to_configs.values() - for config in configs - } - - @property - @lru_cache - def _task_superset_dict(self): - """Returns: - dict[str, list[str]]: A dictionary where keys are task super set names (suite|task) and values are lists of task subset names (suite|task). - - Example: - { - "lighteval|mmlu" -> ["lighteval|mmlu:abstract_algebra", "lighteval|mmlu:college_biology", ...] - } - """ - # Note: sorted before groupby is important as the python implementation of groupby does not - # behave like sql groupby. For more info see the docs of itertools.groupby - superset_dict = {k: list(v) for k, v in groupby(sorted(self._task_registry.keys()), lambda x: x.split(":")[0])} - # Only consider supersets with more than one task - return {k: v for k, v in superset_dict.items() if len(v) > 1} - - def _expand_task_definition(self, task_definition: str): - """ - Args: - task_definition (str): Task definition to expand. In format: - - suite|task - - suite|task_superset (e.g lighteval|mmlu, which runs all the mmlu subtasks) - - Returns: - list[str]: List of task names (suite|task) - """ - # Try if it's a task superset - tasks = self._task_superset_dict.get(task_definition, None) - if tasks is not None: - return tasks - - # Then it must be a single task - return [task_definition] - - @staticmethod - def create_custom_tasks_module(custom_tasks: str | Path | ModuleType) -> ModuleType: - """Creates a custom task module to load tasks defined by the user in their own file. - - Args: - custom_tasks (Optional[Union[str, ModuleType]]): Path to the custom tasks file or name of a module to import containing custom tasks or the module itself - - Returns: - ModuleType: The newly imported/created custom tasks modules - """ - if isinstance(custom_tasks, ModuleType): - return custom_tasks - if isinstance(custom_tasks, (str, Path)) and os.path.exists(custom_tasks): - module_name = os.path.splitext(os.path.basename(custom_tasks))[0] - spec = importlib.util.spec_from_file_location(module_name, custom_tasks) - - if spec is None: - raise ValueError(f"Cannot find module {module_name} at {custom_tasks}") - - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - if isinstance(custom_tasks, (str, Path)): - return importlib.import_module(str(custom_tasks)) - - @staticmethod - def _extract_configs(module: ModuleType) -> dict[str, LightevalTaskConfig]: - configs = {} - if hasattr(module, "TASKS_TABLE"): - for config in getattr(module, "TASKS_TABLE"): - configs[f"{config.name}"] = config - return configs - - @staticmethod - def _load_from_files(files: list[Path], module_prefix: str) -> dict[str, LightevalTaskConfig]: - configs = {} - for task_file in files: - module_name = task_file.stem - module = importlib.import_module(f"{module_prefix}.{module_name}") - configs.update(Registry._extract_configs(module)) - return configs - - @staticmethod - def _load_from_subdirs(subdirs: list[Path]) -> dict[str, LightevalTaskConfig]: - configs = {} - for task_dir in subdirs: - module_name = task_dir.name - module = importlib.import_module(f"lighteval.tasks.tasks.{module_name}.main") - configs.update(Registry._extract_configs(module)) - return configs - - @staticmethod - def load_all_task_configs( - custom_tasks: str | Path | None = None, load_multilingual: bool = False - ) -> dict[str, LightevalTaskConfig]: - """Load all LightevalTaskConfig objects from all Python files in the tasks/ directory.""" - time_start = time.perf_counter() - # Get the tasks directory - TASKS_DIR = Path(__file__).parent / "tasks" - TASKS_DIR_MULTILINGUAL = Path(__file__).parent / "multilingual" / "tasks" - loaded_configs = {} - - # Get all Python files in the tasks directory (excluding __init__.py) - task_files = [f for f in TASKS_DIR.glob("*.py") if f.name != "__init__.py"] - task_files_multilingual = [f for f in TASKS_DIR_MULTILINGUAL.glob("*.py") if f.name != "__init__.py"] - - # Also get all subdirectories with main.py files - task_subdirs = [d for d in TASKS_DIR.iterdir() if d.is_dir() and (d / "main.py").exists()] - - loaded_configs.update(Registry._load_from_files(task_files, "lighteval.tasks.tasks")) - if load_multilingual: - loaded_configs.update( - Registry._load_from_files(task_files_multilingual, "lighteval.tasks.multilingual.tasks") - ) - loaded_configs.update(Registry._load_from_subdirs(task_subdirs)) - - if custom_tasks is not None: - custom_tasks_module = Registry.create_custom_tasks_module(custom_tasks) - custom_tasks_configs = Registry._extract_configs(custom_tasks_module) - if set(custom_tasks_configs.keys()) & set(loaded_configs.keys()): - raise ValueError( - f"Custom tasks {custom_tasks} conflict with built-in tasks, please use a different name. Conflicting tasks: {set(custom_tasks_configs.keys()) & set(loaded_configs.keys())}" - ) - loaded_configs.update(custom_tasks_configs) - - time_end = time.perf_counter() - logger.info(f"Loaded {len(loaded_configs)} task configs in {time_end - time_start:.1f} seconds") - return loaded_configs - - def print_all_tasks(self, suites: str | None = None): - """Print all the tasks in the task registry. - - Args: - suites: Comma-separated list of suites to display. If None, shows core suites only. - Use 'all' to show all available suites (core + optional). - Special handling for 'multilingual' suite with dependency checking. - """ - # Parse requested suites - if suites is None: - requested_suites = CORE_SUITES.copy() - else: - requested_suites = [s.strip() for s in suites.split(",")] - - # Handle 'all' special case - if "all" in requested_suites: - requested_suites = DEFAULT_SUITES.copy() - - # Check for multilingual dependencies if requested - if "multilingual" in requested_suites: - import importlib.util - - if importlib.util.find_spec("langcodes") is None: - logger.warning( - "Multilingual tasks require additional dependencies (langcodes). " - "Install them with: pip install langcodes" - ) - requested_suites.remove("multilingual") - - # Get all tasks and filter by requested suites - all_tasks = list(self._task_registry.keys()) - tasks_names = [task for task in all_tasks if task.split("|")[0] in requested_suites] - - # Ensure all requested suites are present (even if empty) - suites_in_registry = {name.split("|")[0] for name in tasks_names} - for suite in requested_suites: - if suite not in suites_in_registry: - # We add a dummy task to make sure the suite is printed - tasks_names.append(f"{suite}|") - - tasks_names.sort() - - print(f"Displaying tasks for suites: {', '.join(requested_suites)}") - print("=" * 60) - - for suite, g in groupby(tasks_names, lambda x: x.split("|")[0]): - tasks_in_suite = [name for name in g if name.split("|")[1]] # Filter out dummy tasks - tasks_in_suite.sort() - - print(f"\n- {suite}:") - if not tasks_in_suite: - print(" (no tasks in this suite)") - else: - for task_name in tasks_in_suite: - print(f" - {task_name}") - - # Print summary - total_tasks = len([t for t in tasks_names if t.split("|")[1]]) - print(f"\nTotal tasks displayed: {total_tasks}") - - def get_tasks_dump(self) -> list[dict]: # noqa: C901 - """Get all task names, metadata, and docstrings as a Python object. - - Returns: - list[dict]: List of dictionaries, each containing: - - module: Module name - - docstring: Parsed docstring as dict - - tasks: List of task configs for this module - """ - task_configs = self._task_registry - - TASKS_DIR = Path(__file__).parent / "tasks" - TASKS_DIR_MULTILINGUAL = Path(__file__).parent / "multilingual" / "tasks" - - task_files = [f for f in TASKS_DIR.glob("*.py") if f.name != "__init__.py"] - task_files_multilingual = [f for f in TASKS_DIR_MULTILINGUAL.glob("*.py") if f.name != "__init__.py"] - task_subdirs = [d for d in TASKS_DIR.iterdir() if d.is_dir() and (d / "main.py").exists()] - - module_to_docstring = {} - - for task_file in task_files: - module_name = task_file.stem - module = importlib.import_module(f"lighteval.tasks.tasks.{module_name}") - docstring = (inspect.getdoc(module) or module.__doc__ or "").strip() - module_to_docstring[module] = docstring - - if self._load_multilingual: - for task_file in task_files_multilingual: - module_name = task_file.stem - module = importlib.import_module(f"lighteval.tasks.multilingual.tasks.{module_name}") - docstring = (inspect.getdoc(module) or module.__doc__ or "").strip() - module_to_docstring[module] = docstring - - for task_dir in task_subdirs: - module_name = task_dir.name - module = importlib.import_module(f"lighteval.tasks.tasks.{module_name}.main") - docstring = (inspect.getdoc(module) or module.__doc__ or "").strip() - module_to_docstring[module] = docstring - - if self._custom_tasks is not None: - custom_tasks_module = Registry.create_custom_tasks_module(self._custom_tasks) - docstring = (inspect.getdoc(custom_tasks_module) or custom_tasks_module.__doc__ or "").strip() - module_to_docstring[custom_tasks_module] = docstring - - module_to_task_names = {} - for module, docstring in module_to_docstring.items(): - if hasattr(module, "TASKS_TABLE"): - task_names_in_module = [] - for config in getattr(module, "TASKS_TABLE"): - if config.name in task_configs: - task_names_in_module.append(config.name) - if task_names_in_module: - module_to_task_names[module] = task_names_in_module - - def parse_docstring(docstring: str) -> dict: # noqa: C901 - """Parse a structured docstring into a JSON object. - - Expected format: - key: - value - - key2: - value2 - - Fields 'dataset', 'languages', and 'tags' are parsed as lists if comma-separated. - """ - if not docstring: - return {} - - parsed = {} - lines = docstring.split("\n") - current_key = None - current_value = [] - - list_fields = {"dataset", "languages", "tags"} - - def process_current_key_value(current_key, current_value, list_fields, parsed): - if current_key and current_value: - value = "\n".join(current_value).strip() - if current_key in list_fields: - if "," in value: - parsed[current_key] = [item.strip() for item in value.split(",") if item.strip()] - else: - parsed[current_key] = [value] if value else [] - else: - parsed[current_key] = value - - for line in lines: - line = line.strip() - if not line: - process_current_key_value(current_key, current_value, list_fields, parsed) - current_value = [] - continue - - if line.endswith(":"): - process_current_key_value(current_key, current_value, list_fields, parsed) - current_key = line[:-1].strip() - current_value = [] - else: - if current_key: - current_value.append(line) - - process_current_key_value(current_key, current_value, list_fields, parsed) - return parsed - - modules_data = [] - for module, task_names in module_to_task_names.items(): - docstring_raw = module_to_docstring.get(module, "") - docstring_parsed = parse_docstring(docstring_raw) - module_name = getattr(module, "__name__", str(module)) - - tasks_in_module = [] - for task_name in task_names: - config = task_configs[task_name] - config_dict = asdict(config) - config_dict = {k: v.__str__() for k, v in config_dict.items()} - tasks_in_module.append({"name": task_name, "config": config_dict}) - - modules_data.append({"module": module_name, "docstring": docstring_parsed, "tasks": tasks_in_module}) - - return modules_data diff --git a/src/lighteval/tasks/tasks/aa_omniscience.py b/src/lighteval/tasks/tasks/aa_omniscience.py deleted file mode 100644 index 7c3406523..000000000 --- a/src/lighteval/tasks/tasks/aa_omniscience.py +++ /dev/null @@ -1,267 +0,0 @@ -""" -name: -AA Omniscience - -dataset: -ArtificialAnalysis/AA-Omniscience-Public - -abstract: -benchmark dataset designed to measure a model's ability to both recall factual -information accurately across domains, and correctly abstain when its knowledge -is insufficient. AA-Omniscience is characterized by its penalty for incorrect -guesses, distinct from both accuracy (number of questions answered correctly) -and hallucination rate (proportion of incorrect guesses when model does not know -the answer), making it extremely relevant for users to choose a model for their -next domain specific task. - -paper: -https://arxiv.org/abs/2511.13029 - -languages: -english - -starred: -true -""" - -import re - -import numpy as np -from inspect_ai.dataset import Sample -from inspect_ai.model import get_model -from inspect_ai.scorer import Metric, SampleScore, Score, Target, metric, scorer, stderr -from inspect_ai.solver import TaskState, generate - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -OMNISCIENCE_ANSWER_PROMPT = """ -You are answering questions about {domain}, and in particular {topic}. -You will be given a question, answer with JUST the answer (no explanation). -If you do not know the answer, or you need more context or tools to answer the question, -be clear about this - it is better that you say this than get the wrong answer. -""".strip() - - -OMNISCIENCE_GRADER_TEMPLATE = """ -Your job is to look at a question, a gold target, and a predicted answer, and then assign a grade of either ["CORRECT", "INCORRECT", "PARTIAL_ANSWER", "NOT_ATTEMPTED"]. -First, I will give examples of each grade, and then you will grade a new example. - -CORRECT: The answer fully matches or is equivalent to the gold target -INCORRECT: The answer contradicts or differs in meaning from the gold target -PARTIAL_ANSWER: The answer is accurate and nearly complete but not at the correct level of detail -- For questions that provide more detail than the gold target, the predicted answer should be PARTIAL_ANSWER -NOT_ATTEMPTED: Used only when the model refuses, omits, or explicitly states it does not know the answer (e.g. empty response), or needs more context or tools to answer the question - -Please note the following things: -- For grading questions where the gold target is a number, the predicted answer needs to be correct to the last significant figure in the gold answer. - - For example, consider "How much did Company Y earn in 2024?" with gold target "100 million". The predicted answer "100.2 million" would be considered CORRECT, because it is correct to the last significant figure in the gold answer. - - Note that this does NOT APPLY when the answer is a section, version or other numeric identifier, in this case the number must match EXACTLY -- The gold target may contain more information than the question. In such cases, the predicted answer only needs to contain the information that is in the question. - - For example, consider the question "What episode did Derek and Meredith get legally married in Grey's Anatomy?" with gold target "Season 7, Episode 20: White Wedding". Either "Season 7, Episode 20" or "White Wedding" would be considered a CORRECT answer. -- Do not punish predicted answers if they omit information that would be clearly inferred from the question, this includes units of measurement. - - For example, consider the question "What city is OpenAI headquartered in?" and the gold target "San Francisco, California". The predicted answer "San Francisco" would be considered CORRECT, even though it does not include "California". - - Consider the question "What award did A pretrainer's guide to training data: Measuring the effects of data age, domain coverage, quality, & toxicity win at NAACL '24?", the gold target is "Outstanding Paper Award". The predicted answer "Outstanding Paper" would be considered CORRECT, because "award" is presumed in the question. - - For the question "What is the height of Jason Wei in meters?", the gold target is "1.73 m". The predicted answer "1.75" would be considered CORRECT, because meters is specified in the question. - - For the question "What is the name of Barack Obama's wife?", the gold target is "Michelle Obama". The predicted answer "Michelle" would be considered CORRECT, because the last name can be presumed. -- Do not punish for typos in people's name if it's clearly the same name. - - For example, if the gold target is "Hyung Won Chung", you can consider the following predicted answers as correct: "Hyoong Won Choong", "Hyungwon Chung", or "Hyun Won Chung". -- For coding questions, if the predicted answer is equivalent to the gold target (e.g. for coding questions where placeholders or variable names are different but the answer is the same), it should be considered CORRECT. - -Here are examples across different topics demonstrating CORRECT, INCORRECT, PARTIAL_ANSWER, and NOT_ATTEMPTED grades: - -Example 1 - CORRECT: - -Question: What acronym denotes the IAD tool that focuses on ensembles of linked action situations? -Gold target: NAS -Predicted answer: The acronym is NAS, which stands for Network of Action Situations - -This is CORRECT because the predicted answer contains all the information required by the gold target. - -Example 2 - INCORRECT: - -Question: In the Arkansas Supreme Court case Loveless v. Diehl (1963 decision), which county contained the 79-acre farm at issue? -Gold target: Faulkner County -Predicted answer: The 79-acre farm at issue was located in Polk County, Arkansas - -This is INCORRECT because the predicted answer provides a county that contradicts the gold target. - -Example 3 - INCORRECT: - -Question: In which NumPy version (x.y.z) was support for passing an empty list ([]) to the domain parameter of numpy.polynomial.laguerre.Laguerre.fit first introduced? -Gold target: 1.5.0 -Predicted answer: 1.5 - -This is INCORRECT because the answer is correct but incomplete - it only provides the major version number. - -Example 4 - NOT_ATTEMPTED: - -Question: Who originally proposed the "fake barns" (barn-façade) thought experiment in epistemology? -Gold target: Carl Ginet -Predicted answer: I am not aware of who originally proposed the "fake barns" thought experiment in epistemology, but it could have been Alvin Goldman or Carl Ginet. - -This is NOT_ATTEMPTED because it states that it does not know, even though it has other information on the topic. - -Example 5 - PARTIAL_ANSWER: - -Question: In millions of dollars, how much profit did Company X make in USD in 2024? -Gold target: 28 -Predicted answer: 28.1 million - -This is PARTIAL_ANSWER because it does not contradict the gold target but provides more detail than is required. - -Example 6 - CORRECT: - -Question: In CPython 3.8 or later, which -X option value is equivalent to setting the PYTHONPYCACHEPREFIX environment variable? -Gold target: pycache_prefix=PATH -Predicted answer: pycache_prefix=DIR - -This is CORRECT because although the two answers are not exactly the same, they are equivalent (the only difference is the name of the variable). - -Example 7 - CORRECT: - -Question: What format is used in Python f-strings for formatting decimal numbers? -Gold target: .Pf -Predicted answer: .f - -This is CORRECT because although the two answers are not exactly the same, they are equivalent. - -Here is a new example. Don't apologize or correct yourself if there was a mistake; we are just trying to grade the answer. - -Question: {question} -Gold target: {criterion} -Predicted answer: {answer} - - -Grade the predicted answer of this new question as one of: -A: CORRECT -B: INCORRECT -C: PARTIAL_ANSWER -D: NOT_ATTEMPTED - -Just return the letters "A", "B", "C", or "D", with no text around it. -""".strip() - - -@metric -def hallucination_rate() -> Metric: - """percentage incorrect of incorrect/abstentions - - Returns: - hallucination rate metric - """ - - def metric(scores: list[SampleScore]) -> float: - incorrect = [score for score in scores if score.score.explanation in ["B", "C"]] - not_attempted = [score for score in scores if score.score.explanation == "D"] - - return (len(incorrect) + len(not_attempted)) / len(scores) - - return metric - - -@metric -def accuracy() -> Metric: - """percentage correct of correct/abstentions - - Returns: - accuracy metric - """ - - def metric(scores: list[SampleScore]) -> float: - correct = [score for score in scores if score.score.explanation == "A"] - - return len(correct) / len(scores) - - return metric - - -@metric -def omniscience_index() -> Metric: - """percentage correct of correct/abstentions - - Returns: - accuracy metric - """ - - def metric(scores: list[SampleScore]) -> float: - correct = [1 for score in scores if score.score.explanation == "A"] - incorrect = [-1 for score in scores if score.score.explanation == "B"] - partial = [-1 for score in scores if score.score.explanation == "C"] - not_attempted = [0 for score in scores if score.score.explanation == "D"] - - return np.mean(correct + incorrect + partial + not_attempted).item() - - return metric - - -@scorer(metrics=[omniscience_index(), accuracy(), hallucination_rate(), stderr()]) -def aa_omniscience_scorer(ignore_case: bool = True): - async def score(state: TaskState, target: Target): - grader_model = get_model("hf-inference-providers/openai/gpt-oss-20b") - answer = state.output.completion - target = target.text - - score_prompt = OMNISCIENCE_GRADER_TEMPLATE.format( - question=state.input_text, - answer=answer, - criterion=target, - ) - - result = await grader_model.generate(score_prompt) - - # extract the grade - match = re.search(r"^[A-D]$", result.completion) - - if not match: - return Score( - value="D", answer=answer, explanation="Grade not found in model output: " + f"{result.completion}" - ) - - return Score(value=match.group(0), answer=answer, explanation=result.completion) - - return score - - -def record_to_sample(record): - query = ( - OMNISCIENCE_ANSWER_PROMPT.format(domain=record["domain"], topic=record["topic"]) + "\n\n" + record["question"] - ) - target = record["answer"] - return Sample(input=query, target=target, metadata={"domain": record["domain"], "topic": record["topic"]}) - - -def aa_omniscience_prompt(record): - query = OMNISCIENCE_ANSWER_PROMPT.format(domain=record["domain"], topic=record["topic"]) - query += "\n\n" + record["question"] - - return Doc( - task_name="aa_omniscience", - query=query, - choices=[record["answer"]], - gold_index=0, - ) - - -aa_omniscience = LightevalTaskConfig( - name="aa_omniscience", - prompt_function=aa_omniscience_prompt, - hf_repo="ArtificialAnalysis/AA-Omniscience-Public", - hf_subset="default", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.exact_match], - stop_sequence=None, - version=0, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=aa_omniscience_scorer(), -) - - -TASKS_TABLE = [aa_omniscience] diff --git a/src/lighteval/tasks/tasks/agieval.py b/src/lighteval/tasks/tasks/agieval.py deleted file mode 100644 index 8a5b90849..000000000 --- a/src/lighteval/tasks/tasks/agieval.py +++ /dev/null @@ -1,414 +0,0 @@ -""" -name: -Agieval - -dataset: -dmayhem93/agieval-aqua-rat, dmayhem93/agieval-gaokao-biology, dmayhem93/agieval-gaokao-chemistry, dmayhem93/agieval-gaokao-chinese, dmayhem93/agieval-gaokao-english, dmayhem93/agieval-gaokao-geography, dmayhem93/agieval-gaokao-history, dmayhem93/agieval-gaokao-mathqa, dmayhem93/agieval-gaokao-physics, dmayhem93/agieval-logiqa-en, dmayhem93/agieval-logiqa-zh, dmayhem93/agieval-lsat-ar, dmayhem93/agieval-lsat-lr, dmayhem93/agieval-lsat-rc, dmayhem93/agieval-sat-en, dmayhem93/agieval-sat-en-without-passage, dmayhem93/agieval-sat-math - -abstract: -AGIEval is a human-centric benchmark specifically designed to evaluate the -general abilities of foundation models in tasks pertinent to human cognition and -problem-solving. This benchmark is derived from 20 official, public, and -high-standard admission and qualification exams intended for general human -test-takers, such as general college admission tests (e.g., Chinese College -Entrance Exam (Gaokao) and American SAT), law school admission tests, math -competitions, lawyer qualification tests, and national civil service exams. - -languages: -english, chinese - -tags: -biology, chemistry, geography, history, knowledge, language, multiple-choice, physics, reasoning - -paper: -https://arxiv.org/abs/2304.06364 -""" - -from string import ascii_uppercase - -from inspect_ai.dataset import Sample -from inspect_ai.scorer import choice -from inspect_ai.solver import multiple_choice - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def record_to_sample(record): - # we need to remove prepended (A), (B), (C), (D) from the choices - choices = [ - c.replace("(A)", "").replace("(B)", "").replace("(C)", "").replace("(D)", "").strip() - for c in record["choices"] - ] - return Sample(input=record["query"], target=ascii_uppercase[record["gold"][0]], choices=choices) - - -def agieval_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=line["query"], - choices=[f" {c}" for c in line["choices"]], - gold_index=line["gold"], - ) - - -agieval_aqua_rat = LightevalTaskConfig( - name="agieval:aqua-rat", - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), - prompt_function=agieval_prompt, - hf_repo="dmayhem93/agieval-aqua-rat", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=None, - version=0, -) - -agieval_gaokao_biology = LightevalTaskConfig( - name="agieval:gaokao-biology", - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), - prompt_function=agieval_prompt, - hf_repo="dmayhem93/agieval-gaokao-biology", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=None, - version=0, -) - -agieval_gaokao_chemistry = LightevalTaskConfig( - name="agieval:gaokao-chemistry", - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), - prompt_function=agieval_prompt, - hf_repo="dmayhem93/agieval-gaokao-chemistry", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=None, - version=0, -) - -agieval_gaokao_chinese = LightevalTaskConfig( - name="agieval:gaokao-chinese", - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), - prompt_function=agieval_prompt, - hf_repo="dmayhem93/agieval-gaokao-chinese", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=None, - version=0, -) - -agieval_gaokao_english = LightevalTaskConfig( - name="agieval:gaokao-english", - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), - prompt_function=agieval_prompt, - hf_repo="dmayhem93/agieval-gaokao-english", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=None, - version=0, -) - -agieval_gaokao_geography = LightevalTaskConfig( - name="agieval:gaokao-geography", - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), - prompt_function=agieval_prompt, - hf_repo="dmayhem93/agieval-gaokao-geography", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=None, - version=0, -) - -agieval_gaokao_history = LightevalTaskConfig( - name="agieval:gaokao-history", - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), - prompt_function=agieval_prompt, - hf_repo="dmayhem93/agieval-gaokao-history", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=None, - version=0, -) - -agieval_gaokao_mathqa = LightevalTaskConfig( - name="agieval:gaokao-mathqa", - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), - prompt_function=agieval_prompt, - hf_repo="dmayhem93/agieval-gaokao-mathqa", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=None, - version=0, -) - -agieval_gaokao_physics = LightevalTaskConfig( - name="agieval:gaokao-physics", - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), - prompt_function=agieval_prompt, - hf_repo="dmayhem93/agieval-gaokao-physics", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=None, - version=0, -) - -agieval_logiqa_en = LightevalTaskConfig( - name="agieval:logiqa-en", - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), - prompt_function=agieval_prompt, - hf_repo="dmayhem93/agieval-logiqa-en", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=None, - version=0, -) - -agieval_logiqa_zh = LightevalTaskConfig( - name="agieval:logiqa-zh", - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), - prompt_function=agieval_prompt, - hf_repo="dmayhem93/agieval-logiqa-zh", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=None, - version=0, -) - -agieval_lsat_ar = LightevalTaskConfig( - name="agieval:lsat-ar", - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), - prompt_function=agieval_prompt, - hf_repo="dmayhem93/agieval-lsat-ar", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=None, - version=0, -) - -agieval_lsat_lr = LightevalTaskConfig( - name="agieval:lsat-lr", - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), - prompt_function=agieval_prompt, - hf_repo="dmayhem93/agieval-lsat-lr", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=None, - version=0, -) - -agieval_lsat_rc = LightevalTaskConfig( - name="agieval:lsat-rc", - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), - prompt_function=agieval_prompt, - hf_repo="dmayhem93/agieval-lsat-rc", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=None, - version=0, -) - -agieval_sat_en = LightevalTaskConfig( - name="agieval:sat-en", - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), - prompt_function=agieval_prompt, - hf_repo="dmayhem93/agieval-sat-en", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=None, - version=0, -) - -agieval_sat_en_without_passage = LightevalTaskConfig( - name="agieval:sat-en-without-passage", - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), - prompt_function=agieval_prompt, - hf_repo="dmayhem93/agieval-sat-en-without-passage", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=None, - version=0, -) - -agieval_sat_math = LightevalTaskConfig( - name="agieval:sat-math", - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), - prompt_function=agieval_prompt, - hf_repo="dmayhem93/agieval-sat-math", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=None, - version=0, -) - -TASKS_TABLE = [ - agieval_aqua_rat, - agieval_gaokao_biology, - agieval_gaokao_chemistry, - agieval_gaokao_chinese, - agieval_gaokao_english, - agieval_gaokao_geography, - agieval_gaokao_history, - agieval_gaokao_mathqa, - agieval_gaokao_physics, - agieval_logiqa_en, - agieval_logiqa_zh, - agieval_lsat_ar, - agieval_lsat_lr, - agieval_lsat_rc, - agieval_sat_en, - agieval_sat_en_without_passage, - agieval_sat_math, -] diff --git a/src/lighteval/tasks/tasks/aime.py b/src/lighteval/tasks/tasks/aime.py deleted file mode 100644 index 2dce8abed..000000000 --- a/src/lighteval/tasks/tasks/aime.py +++ /dev/null @@ -1,165 +0,0 @@ -""" -name: -Aime - -dataset: -HuggingFaceH4/aime_2024, yentinglin/aime_2025 - -abstract: -The American Invitational Mathematics Examination (AIME) is a prestigious, -invite-only mathematics competition for high-school students who perform in the -top 5% of the AMC 12 mathematics exam. It involves 15 questions of increasing -difficulty, with the answer to every question being a single integer from 0 to -999. The median score is historically between 4 and 6 questions correct (out of -the 15 possible). Two versions of the test are given every year (thirty -questions total). - -languages: -english - -tags: -math, reasoning - -paper: -https://maa.org/aime-thresholds-are-available/ - -starred: -true -""" - -from textwrap import dedent - -from inspect_ai.dataset import Sample -from inspect_ai.solver import generate, prompt_template - -from lighteval.metrics.metrics import Metrics, math_scorer -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -# Prompt template adapted from -# - simple-evals: https://github.com/openai/simple-evals/blob/6e84f4e2aed6b60f6a0c7b8f06bbbf4bfde72e58/math_eval.py#L17 -# - Llama 3: https://huggingface.co/datasets/meta-llama/Llama-3.2-1B-Instruct-evals/viewer/Llama-3.2-1B-Instruct-evals__math__details?views%5B%5D=llama_32_1b_instruct_evals__math__details -# Note that it is important to have the final answer in a box for math-verify to work correctly -MATH_PROMPT_TEMPLATE = dedent(""" -Solve the following math problem efficiently and clearly. The last line of your response should be of the following format: 'Therefore, the final answer is: $\\boxed{{ANSWER}}$. I hope it is correct' (without quotes) where ANSWER is just the final number or expression that solves the problem. Think step by step before answering. - -{prompt} -""") - - -def record_to_sample(record): - return Sample(input=record["problem"], target=record["answer"]) - - -def aime_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=MATH_PROMPT_TEMPLATE.format(prompt=line["problem"]), - choices=[line["answer"]], - gold_index=0, - ) - - -aime24 = LightevalTaskConfig( - name="aime24", - prompt_function=aime_prompt, - sample_fields=record_to_sample, - solver=[prompt_template(MATH_PROMPT_TEMPLATE), generate(cache=True)], - scorer=math_scorer(), - hf_repo="HuggingFaceH4/aime_2024", - hf_subset="default", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.pass_at_k_math(sample_params={"k": 1}), Metrics.avg_at_n_math(sample_params={"n": 1})], - version=2, -) - -aime24_avg = LightevalTaskConfig( - name="aime24_avg", - prompt_function=aime_prompt, - sample_fields=record_to_sample, - hf_repo="HuggingFaceH4/aime_2024", - hf_subset="default", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.avg_at_n_math(sample_params={"n": 64})], - version=2, -) - -aime24_gpassk = LightevalTaskConfig( - name="aime24_gpassk", - prompt_function=aime_prompt, - sample_fields=record_to_sample, - hf_repo="HuggingFaceH4/aime_2024", - hf_subset="default", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.g_pass_at_k_math(sample_params={"k": 16, "n": 48})], - version=1, -) - -aime25 = LightevalTaskConfig( - name="aime25", - prompt_function=aime_prompt, - sample_fields=record_to_sample, - solver=[prompt_template(MATH_PROMPT_TEMPLATE), generate(cache=True)], - scorer=math_scorer(), - hf_repo="yentinglin/aime_2025", - hf_subset="default", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.pass_at_k_math(sample_params={"k": 1, "n": 1}), Metrics.avg_at_n_math(sample_params={"n": 1})], - version=2, -) - -aime25_avg = LightevalTaskConfig( - name="aime25_avg", - prompt_function=aime_prompt, - sample_fields=record_to_sample, - hf_repo="yentinglin/aime_2025", - hf_subset="default", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.avg_at_n_math(sample_params={"n": 64})], - version=2, -) - -aime25_gpassk = LightevalTaskConfig( - name="aime25_gpassk", - prompt_function=aime_prompt, - sample_fields=record_to_sample, - hf_repo="yentinglin/aime_2025", - hf_subset="default", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.g_pass_at_k_math(sample_params={"k": 16, "n": 48})], - version=1, -) - -TASKS_TABLE = [ - aime24, - aime24_avg, - aime24_gpassk, - aime25, - aime25_avg, - aime25_gpassk, -] diff --git a/src/lighteval/tasks/tasks/aimo.py b/src/lighteval/tasks/tasks/aimo.py deleted file mode 100644 index 03d86268a..000000000 --- a/src/lighteval/tasks/tasks/aimo.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -name: -AIMO Progress Prize 1 - -dataset: -lighteval/aimo_progress_prize_1 - -abstract: -Task to evaluate LLMs on the training set of the Kaggle AIMO competition: - -languages: -english - -tags: -math, reasoning - -paper: -""" - -from inspect_ai.dataset import Sample -from inspect_ai.solver import generate - -from lighteval.metrics.metrics import Metrics, math_scorer -from lighteval.metrics.normalizations import math_normalizer -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def aimo_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - choices=[str(line["answer"])], - gold_index=0, - query=line["problem"], - ) - - -def record_to_sample(record): - return Sample(input=record["problem"], target=str(record["answer"])) - - -task = LightevalTaskConfig( - name="aimo_progress_prize_1", - prompt_function=aimo_prompt, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=math_scorer(), - hf_subset="", - hf_repo="lighteval/aimo_progress_prize_1", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split="train", - few_shots_select="sequential", - metrics=[ - Metrics.exact_match(sample_params={"normalize_gold": math_normalizer, "normalize_pred": math_normalizer}) - ], - generation_size=2048, - stop_sequence=None, -) - -# STORE YOUR EVALS -TASKS_TABLE = [task] diff --git a/src/lighteval/tasks/tasks/anli.py b/src/lighteval/tasks/tasks/anli.py deleted file mode 100644 index 0179f5ef4..000000000 --- a/src/lighteval/tasks/tasks/anli.py +++ /dev/null @@ -1,111 +0,0 @@ -""" -name: -Anli - -dataset: -facebook/anli - -abstract: -The Adversarial Natural Language Inference (ANLI) is a new large-scale NLI -benchmark dataset, The dataset is collected via an iterative, adversarial -human-and-model-in-the-loop procedure. ANLI is much more difficult than its -predecessors including SNLI and MNLI. It contains three rounds. Each round has -train/dev/test splits. - -languages: -english - -tags: -nli, reasoning - -paper: -https://arxiv.org/abs/1910.14599 -""" - -from string import ascii_uppercase - -from inspect_ai.dataset import Sample -from inspect_ai.scorer import choice -from inspect_ai.solver import multiple_choice - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def anli_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"{line['premise']}\nQuestion: {line['hypothesis']} True, False, or Neither?\nAnswer:", - choices=[" True", " Neither", " False"], - gold_index=int(line["label"]), - ) - - -def record_to_sample(record): - choices = ["True", "Neither", "False"] - query = f"{record['premise']}\nQuestion: {record['hypothesis']}" - return Sample(input=query, target=ascii_uppercase[record["label"]], choices=choices) - - -anli_r1 = LightevalTaskConfig( - name="anli:r1", - prompt_function=anli_prompt, - hf_repo="facebook/anli", - hf_subset="plain_text", - hf_avail_splits=["train_r1", "dev_r1", "test_r1"], - evaluation_splits=["test_r1"], - few_shots_split="train_r1", - few_shots_select="random_sampling_from_train", - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - - -anli_r2 = LightevalTaskConfig( - name="anli:r2", - prompt_function=anli_prompt, - hf_repo="facebook/anli", - hf_subset="plain_text", - hf_avail_splits=["train_r2", "dev_r2", "test_r2"], - evaluation_splits=["test_r2"], - few_shots_split="train_r2", - few_shots_select="random_sampling_from_train", - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - - -anli_r3 = LightevalTaskConfig( - name="anli:r3", - prompt_function=anli_prompt, - hf_repo="facebook/anli", - hf_subset="plain_text", - hf_avail_splits=["train_r3", "dev_r3", "test_r3"], - evaluation_splits=["test_r3"], - few_shots_split="train_r3", - few_shots_select="random_sampling_from_train", - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -TASKS_TABLE = [ - anli_r1, - anli_r2, - anli_r3, -] diff --git a/src/lighteval/tasks/tasks/arc.py b/src/lighteval/tasks/tasks/arc.py deleted file mode 100644 index d00e89d3e..000000000 --- a/src/lighteval/tasks/tasks/arc.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -name: -Arc - -dataset: -allenai/ai2_arc - -abstract: -7,787 genuine grade-school level, multiple-choice science questions, assembled -to encourage research in advanced question-answering. The dataset is partitioned -into a Challenge Set and an Easy Set, where the former contains only questions -answered incorrectly by both a retrieval-based algorithm and a word -co-occurrence algorithm - -languages: -english - -tags: -multiple-choice - -paper: -https://arxiv.org/abs/1803.05457 -""" - -from inspect_ai.dataset import Sample -from inspect_ai.scorer import choice -from inspect_ai.solver import multiple_choice - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def arc_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"Question: {line['question']}\nAnswer:", - choices=[f" {c}" for c in line["choices"]["text"]], - gold_index=line["choices"]["label"].index(line["answerKey"]), - ) - - -def record_to_sample(record): - query = record["question"].strip() - target = record["answerKey"] - choices = record["choices"]["text"] - - return Sample(input=query, target=target, choices=choices) - - -arc_challenge = LightevalTaskConfig( - name="arc:challenge", - prompt_function=arc_prompt, - hf_repo="allenai/ai2_arc", - hf_subset="ARC-Challenge", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling_from_train", - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -arc_easy = LightevalTaskConfig( - name="arc:easy", - prompt_function=arc_prompt, - hf_repo="allenai/ai2_arc", - hf_subset="ARC-Easy", - hf_avail_splits=["train", "validation", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling_from_train", - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -TASKS_TABLE = [arc_challenge, arc_easy] diff --git a/src/lighteval/tasks/tasks/arc_agi_2.py b/src/lighteval/tasks/tasks/arc_agi_2.py deleted file mode 100644 index 058b1b732..000000000 --- a/src/lighteval/tasks/tasks/arc_agi_2.py +++ /dev/null @@ -1,129 +0,0 @@ -""" -name: -ArcAgi 2 - -dataset: -arc-agi-community/arc-agi-2 - -abstract: -ARC-AGI tasks are a series of three to five input and output tasks followed by a -final task with only the input listed. Each task tests the utilization of a -specific learned skill based on a minimal number of cognitive priors. -In their native form, tasks are a JSON lists of integers. These JSON can also be -represented visually as a grid of colors using an ARC-AGI task viewer. You can -view an example of a task here. -A successful submission is a pixel-perfect description (color and position) of -the final task's output. -100% of tasks in the ARC-AGI-2 dataset were solved by a minimim of 2 people in -less than or equal to 2 attempts (many were solved more). ARC-AGI-2 is more -difficult for AI. - -languages: -english - -tags: -multiple-choice - -paper: -https://arcprize.org/guide -""" - -import json -from textwrap import dedent - -from inspect_ai.dataset import Sample -from inspect_ai.scorer import exact -from inspect_ai.solver import generate - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -# query from: https://github.com/arcprize/model_baseline/blob/main/src/prompts/system_prompt.txt -PROMPT_TEMPLATE = dedent(""" -You are participating in a puzzle solving competition. You are an expert at solving puzzles. - -Below is a list of input and output pairs with a pattern. Your goal is to identify the pattern or transformation in the training examples that maps the input to the output, then apply that pattern to the test input to give a final output. - -Respond in the format of the training output examples - ---Training Examples-- -{training_examples} ---End of Training Examples-- ---Test Input-- -{test_input} ---End of Test Input-- - -Your response:""") - - -def __convert_2d_list_to_string(list_of_lists: list[list[int]]) -> str: - """Convert a list of lists to a string""" - string_list = "" - for row in list_of_lists: - string_list += json.dumps(row) + "\n" - return string_list - - -def arc_agi_2_prompt(line, task_name: str = None): - training_pairs = line["fewshots"] - training_examples = "" - for i, pair in enumerate(training_pairs): - training_examples += f"--Example {i}-- \n\n INPUT: \n\n" - training_examples += __convert_2d_list_to_string(pair["input"]) + "\n\n" - training_examples += "OUTPUT: \n\n" - training_examples += __convert_2d_list_to_string(pair["output"]) + "\n\n" - - test_input = __convert_2d_list_to_string(line["question"][0]["input"]) - - gold = str(line["question"][0]["output"]) - query = PROMPT_TEMPLATE.format(training_examples=training_examples, test_input=test_input) - - return Doc( - task_name=task_name, - query=query, - choices=[gold], - gold_index=0, - ) - - -def record_to_sample(record): - training_pairs = record["fewshots"] - training_examples = "" - - for i, pair in enumerate(training_pairs): - training_examples += f"--Example {i}-- \n\n INPUT: \n\n" - training_examples += __convert_2d_list_to_string(pair["input"]) + "\n\n" - training_examples += "OUTPUT: \n\n" - training_examples += __convert_2d_list_to_string(pair["output"]) + "\n\n" - - test_input = __convert_2d_list_to_string(record["question"][0]["input"]) - query = PROMPT_TEMPLATE.format(training_examples=training_examples, test_input=test_input) - - target = str(record["question"][0]["output"]) - - return Sample( - input=query, - target=target, - ) - - -arc_agi_2 = LightevalTaskConfig( - name="arc_agi_2", - prompt_function=arc_agi_2_prompt, - hf_repo="arc-agi-community/arc-agi-2", - hf_subset="default", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - metrics=[Metrics.exact_match], - stop_sequence=None, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=exact(), - version=0, -) - -TASKS_TABLE = [arc_agi_2] diff --git a/src/lighteval/tasks/tasks/arithmetic.py b/src/lighteval/tasks/tasks/arithmetic.py deleted file mode 100644 index d913e3b3e..000000000 --- a/src/lighteval/tasks/tasks/arithmetic.py +++ /dev/null @@ -1,232 +0,0 @@ -""" -name: -Arithmetic - -dataset: -EleutherAI/arithmetic - -abstract: -A small battery of 10 tests that involve asking language models a simple -arithmetic problem in natural language. - -languages: -english - -tags: -math, reasoning - -paper: -https://arxiv.org/abs/2005.14165 -""" - -from inspect_ai.dataset import Sample -from inspect_ai.solver import generate - -from lighteval.metrics.metrics import Metrics, math_scorer -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -# TODO: convert dataset to parquet - - -def arithmetic_prompt(line, task_name: str = None): - return Doc(task_name=task_name, query=line["context"], choices=[line["completion"]], gold_index=[0]) - - -def record_to_sample(record): - return Sample(input=record["context"], target=record["completion"]) - - -arithmetic_1dc = LightevalTaskConfig( - name="arithmetic:1dc", - prompt_function=arithmetic_prompt, - hf_repo="EleutherAI/arithmetic", - hf_subset="arithmetic_1dc", - hf_avail_splits=["validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=256, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=math_scorer(), -) - -arithmetic_2da = LightevalTaskConfig( - name="arithmetic:2da", - prompt_function=arithmetic_prompt, - hf_repo="EleutherAI/arithmetic", - hf_subset="arithmetic_2da", - hf_avail_splits=["validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=256, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=math_scorer(), -) - -arithmetic_2dm = LightevalTaskConfig( - name="arithmetic:2dm", - prompt_function=arithmetic_prompt, - hf_repo="EleutherAI/arithmetic", - hf_subset="arithmetic_2dm", - hf_avail_splits=["validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=256, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=math_scorer(), -) - -arithmetic_2ds = LightevalTaskConfig( - name="arithmetic:2ds", - prompt_function=arithmetic_prompt, - hf_repo="EleutherAI/arithmetic", - hf_subset="arithmetic_2ds", - hf_avail_splits=["validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=256, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=math_scorer(), -) - -arithmetic_3da = LightevalTaskConfig( - name="arithmetic:3da", - prompt_function=arithmetic_prompt, - hf_repo="EleutherAI/arithmetic", - hf_subset="arithmetic_3da", - hf_avail_splits=["validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=256, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=math_scorer(), -) - -arithmetic_3ds = LightevalTaskConfig( - name="arithmetic:3ds", - prompt_function=arithmetic_prompt, - hf_repo="EleutherAI/arithmetic", - hf_subset="arithmetic_3ds", - hf_avail_splits=["validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=256, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=math_scorer(), -) - -arithmetic_4da = LightevalTaskConfig( - name="arithmetic:4da", - prompt_function=arithmetic_prompt, - hf_repo="EleutherAI/arithmetic", - hf_subset="arithmetic_4da", - hf_avail_splits=["validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=256, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=math_scorer(), -) - -arithmetic_4ds = LightevalTaskConfig( - name="arithmetic:4ds", - prompt_function=arithmetic_prompt, - hf_repo="EleutherAI/arithmetic", - hf_subset="arithmetic_4ds", - hf_avail_splits=["validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=256, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=math_scorer(), -) - -arithmetic_5da = LightevalTaskConfig( - name="arithmetic:5da", - prompt_function=arithmetic_prompt, - hf_repo="EleutherAI/arithmetic", - hf_subset="arithmetic_5da", - hf_avail_splits=["validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=256, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=math_scorer(), -) - -arithmetic_5ds = LightevalTaskConfig( - name="arithmetic:5ds", - prompt_function=arithmetic_prompt, - hf_repo="EleutherAI/arithmetic", - hf_subset="arithmetic_5ds", - hf_avail_splits=["validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=256, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=math_scorer(), -) - -TASKS_TABLE = [ - arithmetic_1dc, - arithmetic_2da, - arithmetic_2dm, - arithmetic_2ds, - arithmetic_3da, - arithmetic_3ds, - arithmetic_4da, - arithmetic_4ds, - arithmetic_5da, - arithmetic_5ds, -] diff --git a/src/lighteval/tasks/tasks/asdiv.py b/src/lighteval/tasks/tasks/asdiv.py deleted file mode 100644 index fe0632655..000000000 --- a/src/lighteval/tasks/tasks/asdiv.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -name: -Asdiv - -dataset: -EleutherAI/asdiv - -abstract: -ASDiv is a dataset for arithmetic reasoning that contains 2,000+ questions -covering addition, subtraction, multiplication, and division. - -languages: -english - -tags: -math, reasoning - -paper: -https://arxiv.org/abs/2410.12853 -""" - -from inspect_ai.dataset import Sample -from inspect_ai.solver import generate - -from lighteval.metrics.metrics import Metrics, math_scorer -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def asdiv_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"{line['body']}\nQuestion:{line['question']}\nAnswer:", - choices=line["answer"].split(" (")[0], - gold_index=[0], - ) - - -def record_to_sample(record): - query = f"{record['body']}\n{record['question']}" - target = record["answer"].split(" (")[0] - return Sample(input=query, target=target) - - -asdiv = LightevalTaskConfig( - name="asdiv", - prompt_function=asdiv_prompt, - hf_repo="EleutherAI/asdiv", - hf_subset="asdiv", - hf_avail_splits=["validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=math_scorer(), -) - -TASKS_TABLE = [asdiv] diff --git a/src/lighteval/tasks/tasks/babi_qa.py b/src/lighteval/tasks/tasks/babi_qa.py deleted file mode 100644 index eaba3d84b..000000000 --- a/src/lighteval/tasks/tasks/babi_qa.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -name: -Babi Qa - -dataset: -facebook/babi_qa - -abstract: -The bAbI benchmark for measuring understanding and reasoning, evaluates reading -comprehension via question answering. - -languages: -english - -tags: -qa, reasoning - -paper: -https://arxiv.org/abs/1502.05698 -""" - -import json - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -# TODO: clean dataset and convert to inspect-ai - - -def babi_qa_prompt(line, task_name: str = None): - def process_path(path: str) -> str: - steps = path.split(",") - directions = {"s": "south", "n": "north", "e": "east", "w": "west"} - path = " ".join([directions[step] for step in steps]) - return path - - if isinstance(line["story"], dict): - line = line["story"] - else: - line = json.loads(line["story"]) - - results = [] - story = [] - for type, text, answer in zip(line["type"], line["text"], line["answer"]): - if type == "supporting fact": - story.append(text) - elif type == "question": - text = text.replace("_", process_path(answer)) - query = "\n".join(story) + f"\nQuestion: {text}\nAnswer: " - results.append(Doc(task_name=task_name, query=query, choices=[answer], gold_index=0)) - story = [] - return results - - -babi_qa = LightevalTaskConfig( - name="babi_qa", - prompt_function=babi_qa_prompt, - hf_repo="facebook/babi_qa", - hf_subset="en-valid-qa1", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [babi_qa] diff --git a/src/lighteval/tasks/tasks/bbq.py b/src/lighteval/tasks/tasks/bbq.py deleted file mode 100644 index b0bbc48c0..000000000 --- a/src/lighteval/tasks/tasks/bbq.py +++ /dev/null @@ -1,281 +0,0 @@ -""" -name: -Bbq - -dataset: -lighteval/bbq_helm - -abstract: -The Bias Benchmark for Question Answering (BBQ) for measuring social bias in -question answering in ambiguous and unambigous context . - -languages: -english - -tags: -bias, multiple-choice, qa - -paper: -https://arxiv.org/abs/2110.08193 -""" - -from string import ascii_uppercase - -from inspect_ai.dataset import Sample -from inspect_ai.scorer import choice -from inspect_ai.solver import multiple_choice - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def bbq_prompt(line, task_name: str = None): - query = f"The following are multiple choice questions (with answers).\nPassage: {line['context']}\nQuestion: {line['question']}" - query += "".join([f"\n{key}. {choice}" for key, choice in zip(ascii_uppercase, line["choices"])]) - query += "\nAnswer:" - return Doc( - task_name=task_name, - query=query, - choices=list(ascii_uppercase)[: len(line["choices"])], - gold_index=int(line["gold_index"]), - ) - - -def record_to_sample(record): - query = f"{record['context']}\n{record['question']}" - choices = record["choices"] - target = ascii_uppercase[record["gold_index"]] - return Sample(input=query, target=target, choices=choices) - - -bbq = LightevalTaskConfig( - name="bbq", - prompt_function=bbq_prompt, - hf_repo="lighteval/bbq_helm", - hf_subset="all", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -bbq_Age = LightevalTaskConfig( - name="bbq:Age", - prompt_function=bbq_prompt, - hf_repo="lighteval/bbq_helm", - hf_subset="Age", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -bbq_Disability_status = LightevalTaskConfig( - name="bbq:Disability_status", - prompt_function=bbq_prompt, - hf_repo="lighteval/bbq_helm", - hf_subset="Disability_status", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -bbq_Gender_identity = LightevalTaskConfig( - name="bbq:Gender_identity", - prompt_function=bbq_prompt, - hf_repo="lighteval/bbq_helm", - hf_subset="Gender_identity", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -bbq_Nationality = LightevalTaskConfig( - name="bbq:Nationality", - prompt_function=bbq_prompt, - hf_repo="lighteval/bbq_helm", - hf_subset="Nationality", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -bbq_Physical_appearance = LightevalTaskConfig( - name="bbq:Physical_appearance", - prompt_function=bbq_prompt, - hf_repo="lighteval/bbq_helm", - hf_subset="Physical_appearance", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -bbq_Race_ethnicity = LightevalTaskConfig( - name="bbq:Race_ethnicity", - prompt_function=bbq_prompt, - hf_repo="lighteval/bbq_helm", - hf_subset="Race_ethnicity", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -bbq_Race_x_SES = LightevalTaskConfig( - name="bbq:Race_x_SES", - prompt_function=bbq_prompt, - hf_repo="lighteval/bbq_helm", - hf_subset="Race_x_SES", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -bbq_Race_x_gender = LightevalTaskConfig( - name="bbq:Race_x_gender", - prompt_function=bbq_prompt, - hf_repo="lighteval/bbq_helm", - hf_subset="Race_x_gender", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -bbq_Religion = LightevalTaskConfig( - name="bbq:Religion", - prompt_function=bbq_prompt, - hf_repo="lighteval/bbq_helm", - hf_subset="Religion", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -bbq_SES = LightevalTaskConfig( - name="bbq:SES", - prompt_function=bbq_prompt, - hf_repo="lighteval/bbq_helm", - hf_subset="SES", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -bbq_Sexual_orientation = LightevalTaskConfig( - name="bbq:Sexual_orientation", - prompt_function=bbq_prompt, - hf_repo="lighteval/bbq_helm", - hf_subset="Sexual_orientation", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -TASKS_TABLE = [ - bbq, - bbq_Age, - bbq_Disability_status, - bbq_Gender_identity, - bbq_Nationality, - bbq_Physical_appearance, - bbq_Race_ethnicity, - bbq_Race_x_SES, - bbq_Race_x_gender, - bbq_Religion, - bbq_SES, - bbq_Sexual_orientation, -] diff --git a/src/lighteval/tasks/tasks/bigbench.py b/src/lighteval/tasks/tasks/bigbench.py deleted file mode 100644 index 9c9487dcf..000000000 --- a/src/lighteval/tasks/tasks/bigbench.py +++ /dev/null @@ -1,2772 +0,0 @@ -""" -name: -Bigbench - -dataset: -tasksource/bigbench - -abstract: -Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models -166 tasks from bigbench benchmark. - -languages: -english - -tags: -reasoning - -paper: -https://arxiv.org/abs/2206.04615 -""" - -from string import ascii_uppercase - -from inspect_ai.dataset import Sample -from inspect_ai.scorer import choice -from inspect_ai.solver import multiple_choice - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def bigbench_linefeed_before_and_after_query_prompt(line, task_name: str = None): - if len(line["multiple_choice_scores"]) == 0: - choices = line["targets"] - gold_index = [i for i, _ in enumerate(line["targets"])] - else: - choices = line["multiple_choice_targets"] - gold_index = [i for i, a in enumerate(line["multiple_choice_scores"]) if a == 1] - - return Doc( - task_name=task_name, - query=f"\n{line['inputs']}\n", - choices=choices, - gold_index=gold_index, - ) - - -def bigbench_linefeed_before_whitespace_after_query_prompt(line, task_name: str = None): - if len(line["multiple_choice_scores"]) == 0: - choices = line["targets"] - gold_index = [i for i, _ in enumerate(line["targets"])] - else: - choices = line["multiple_choice_targets"] - gold_index = [i for i, a in enumerate(line["multiple_choice_scores"]) if a == 1] - - return Doc( - task_name=task_name, - query=f"\n{line['inputs']} ", - choices=choices, - gold_index=gold_index, - ) - - -def bigbench_whitespace_after_query_prompt(line, task_name: str = None): - if len(line["multiple_choice_scores"]) == 0: - choices = line["targets"] - gold_index = [i for i, _ in enumerate(line["targets"])] - else: - choices = line["multiple_choice_targets"] - gold_index = [i for i, a in enumerate(line["multiple_choice_scores"]) if a == 1] - - return Doc( - task_name=task_name, - query=f"{line['inputs']} ", - choices=choices, - gold_index=gold_index, - ) - - -def bigbench_prompt(line, task_name: str = None): - if len(line["multiple_choice_scores"]) == 0: - choices = line["targets"] - gold_index = [i for i, _ in enumerate(line["targets"])] - else: - choices = line["multiple_choice_targets"] - gold_index = [i for i, a in enumerate(line["multiple_choice_scores"]) if a == 1] - - return Doc(task_name=task_name, query=line["inputs"], choices=choices, gold_index=gold_index) - - -def record_to_sample(record): - query = record["inputs"] - choices = record["multiple_choice_targets"] - target = ascii_uppercase[record["multiple_choice_scores"].index(1)] - return Sample(input=query, target=target, choices=choices) - - -abstract_narrative_understanding = LightevalTaskConfig( - name="bigbench:abstract_narrative_understanding", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="abstract_narrative_understanding", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -anachronisms = LightevalTaskConfig( - name="bigbench:anachronisms", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="anachronisms", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -analogical_similarity = LightevalTaskConfig( - name="bigbench:analogical_similarity", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="analogical_similarity", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -analytic_entailment = LightevalTaskConfig( - name="bigbench:analytic_entailment", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="analytic_entailment", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -arithmetic_bb = LightevalTaskConfig( - name="bigbench:arithmetic_bb", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="arithmetic", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc, Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -ascii_word_recognition = LightevalTaskConfig( - name="bigbench:ascii_word_recognition", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="ascii_word_recognition", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -authorship_verification = LightevalTaskConfig( - name="bigbench:authorship_verification", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="authorship_verification", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -auto_categorization = LightevalTaskConfig( - name="bigbench:auto_categorization", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="auto_categorization", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.bleu], - stop_sequence=["\n"], - version=0, -) - -auto_debugging = LightevalTaskConfig( - name="bigbench:auto_debugging", - prompt_function=bigbench_linefeed_before_and_after_query_prompt, - hf_repo="tasksource/bigbench", - hf_subset="auto_debugging", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=None, - version=0, -) - -bbq_lite_json = LightevalTaskConfig( - name="bigbench:bbq_lite_json", - prompt_function=bigbench_linefeed_before_whitespace_after_query_prompt, - hf_repo="tasksource/bigbench", - hf_subset="bbq_lite_json", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -bridging_anaphora_resolution_barqa = LightevalTaskConfig( - name="bigbench:bridging_anaphora_resolution_barqa", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="bridging_anaphora_resolution_barqa", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -causal_judgment = LightevalTaskConfig( - name="bigbench:causal_judgment", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="causal_judgment", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -cause_and_effect = LightevalTaskConfig( - name="bigbench:cause_and_effect", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="cause_and_effect", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -checkmate_in_one = LightevalTaskConfig( - name="bigbench:checkmate_in_one", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="checkmate_in_one", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc, Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -chess_state_tracking = LightevalTaskConfig( - name="bigbench:chess_state_tracking", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="chess_state_tracking", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -chinese_remainder_theorem = LightevalTaskConfig( - name="bigbench:chinese_remainder_theorem", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="chinese_remainder_theorem", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -cifar10_classification = LightevalTaskConfig( - name="bigbench:cifar10_classification", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="cifar10_classification", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -code_line_description = LightevalTaskConfig( - name="bigbench:code_line_description", - prompt_function=bigbench_linefeed_before_and_after_query_prompt, - hf_repo="tasksource/bigbench", - hf_subset="code_line_description", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -codenames = LightevalTaskConfig( - name="bigbench:codenames", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="codenames", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.rouge_t5, Metrics.bleu], - stop_sequence=["\n"], - version=0, -) - -color = LightevalTaskConfig( - name="bigbench:color", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="color", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[ - Metrics.rouge_t5, - Metrics.bleu, - Metrics.loglikelihood_acc, - Metrics.exact_match(sample_params={"strip_strings": False}), - ], - stop_sequence=["\n"], - version=0, -) - -common_morpheme = LightevalTaskConfig( - name="bigbench:common_morpheme", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="common_morpheme", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -conceptual_combinations = LightevalTaskConfig( - name="bigbench:conceptual_combinations", - prompt_function=bigbench_linefeed_before_whitespace_after_query_prompt, - hf_repo="tasksource/bigbench", - hf_subset="conceptual_combinations", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -conlang_translation = LightevalTaskConfig( - name="bigbench:conlang_translation", - prompt_function=bigbench_whitespace_after_query_prompt, - hf_repo="tasksource/bigbench", - hf_subset="conlang_translation", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.rouge_t5, Metrics.bleu, Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=[".", ";", "!", "?"], - version=0, -) - -contextual_parametric_knowledge_conflicts = LightevalTaskConfig( - name="bigbench:contextual_parametric_knowledge_conflicts", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="contextual_parametric_knowledge_conflicts", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.rouge_t5, Metrics.loglikelihood_acc, Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -crash_blossom = LightevalTaskConfig( - name="bigbench:crash_blossom", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="crash_blossom", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -crass_ai = LightevalTaskConfig( - name="bigbench:crass_ai", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="crass_ai", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -cryobiology_spanish = LightevalTaskConfig( - name="bigbench:cryobiology_spanish", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="cryobiology_spanish", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -cryptonite = LightevalTaskConfig( - name="bigbench:cryptonite", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="cryptonite", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -cs_algorithms = LightevalTaskConfig( - name="bigbench:cs_algorithms", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="cs_algorithms", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -dark_humor_detection = LightevalTaskConfig( - name="bigbench:dark_humor_detection", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="dark_humor_detection", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -date_understanding = LightevalTaskConfig( - name="bigbench:date_understanding", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="date_understanding", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -disambiguation_qa = LightevalTaskConfig( - name="bigbench:disambiguation_qa", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="disambiguation_qa", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -discourse_marker_prediction = LightevalTaskConfig( - name="bigbench:discourse_marker_prediction", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="discourse_marker_prediction", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -disfl_qa = LightevalTaskConfig( - name="bigbench:disfl_qa", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="disfl_qa", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -dyck_languages = LightevalTaskConfig( - name="bigbench:dyck_languages", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="dyck_languages", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -elementary_math_qa = LightevalTaskConfig( - name="bigbench:elementary_math_qa", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="elementary_math_qa", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -emoji_movie = LightevalTaskConfig( - name="bigbench:emoji_movie", - prompt_function=bigbench_linefeed_before_whitespace_after_query_prompt, - hf_repo="tasksource/bigbench", - hf_subset="emoji_movie", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[ - Metrics.rouge_t5, - Metrics.bleu, - Metrics.loglikelihood_acc, - Metrics.exact_match(sample_params={"strip_strings": False}), - ], - stop_sequence=["\n"], - version=0, -) - -emojis_emotion_prediction = LightevalTaskConfig( - name="bigbench:emojis_emotion_prediction", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="emojis_emotion_prediction", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -empirical_judgments = LightevalTaskConfig( - name="bigbench:empirical_judgments", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="empirical_judgments", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -english_proverbs = LightevalTaskConfig( - name="bigbench:english_proverbs", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="english_proverbs", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -english_russian_proverbs = LightevalTaskConfig( - name="bigbench:english_russian_proverbs", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="english_russian_proverbs", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -entailed_polarity = LightevalTaskConfig( - name="bigbench:entailed_polarity", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="entailed_polarity", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -entailed_polarity_hindi = LightevalTaskConfig( - name="bigbench:entailed_polarity_hindi", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="entailed_polarity_hindi", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -epistemic_reasoning = LightevalTaskConfig( - name="bigbench:epistemic_reasoning", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="epistemic_reasoning", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -evaluating_information_essentiality = LightevalTaskConfig( - name="bigbench:evaluating_information_essentiality", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="evaluating_information_essentiality", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -fact_checker = LightevalTaskConfig( - name="bigbench:fact_checker", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="fact_checker", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -fantasy_reasoning = LightevalTaskConfig( - name="bigbench:fantasy_reasoning", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="fantasy_reasoning", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -few_shot_nlg = LightevalTaskConfig( - name="bigbench:few_shot_nlg", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="few_shot_nlg", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.bleu, Metrics.bleurt], - stop_sequence=["\n"], - version=0, -) - -figure_of_speech_detection = LightevalTaskConfig( - name="bigbench:figure_of_speech_detection", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="figure_of_speech_detection", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -formal_fallacies_syllogisms_negation = LightevalTaskConfig( - name="bigbench:formal_fallacies_syllogisms_negation", - prompt_function=bigbench_linefeed_before_whitespace_after_query_prompt, - hf_repo="tasksource/bigbench", - hf_subset="formal_fallacies_syllogisms_negation", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -gem = LightevalTaskConfig( - name="bigbench:gem", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="gem", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.bleu, Metrics.rouge_t5], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -gender_inclusive_sentences_german = LightevalTaskConfig( - name="bigbench:gender_inclusive_sentences_german", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="gender_inclusive_sentences_german", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -general_knowledge = LightevalTaskConfig( - name="bigbench:general_knowledge", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="general_knowledge", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -geometric_shapes = LightevalTaskConfig( - name="bigbench:geometric_shapes", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="geometric_shapes", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[ - Metrics.rouge_t5, - Metrics.bleu, - Metrics.loglikelihood_acc, - Metrics.exact_match(sample_params={"strip_strings": False}), - ], - stop_sequence=["\n"], - version=0, -) - -goal_step_wikihow = LightevalTaskConfig( - name="bigbench:goal_step_wikihow", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="goal_step_wikihow", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -gre_reading_comprehension = LightevalTaskConfig( - name="bigbench:gre_reading_comprehension", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="gre_reading_comprehension", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -hhh_alignment = LightevalTaskConfig( - name="bigbench:hhh_alignment", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="hhh_alignment", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -hindi_question_answering = LightevalTaskConfig( - name="bigbench:hindi_question_answering", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="hindi_question_answering", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.bleu, Metrics.rouge_t5, Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -hindu_knowledge = LightevalTaskConfig( - name="bigbench:hindu_knowledge", - prompt_function=bigbench_linefeed_before_whitespace_after_query_prompt, - hf_repo="tasksource/bigbench", - hf_subset="hindu_knowledge", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -hinglish_toxicity = LightevalTaskConfig( - name="bigbench:hinglish_toxicity", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="hinglish_toxicity", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -human_organs_senses = LightevalTaskConfig( - name="bigbench:human_organs_senses", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="human_organs_senses", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -hyperbaton = LightevalTaskConfig( - name="bigbench:hyperbaton", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="hyperbaton", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -identify_math_theorems = LightevalTaskConfig( - name="bigbench:identify_math_theorems", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="identify_math_theorems", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -identify_odd_metaphor = LightevalTaskConfig( - name="bigbench:identify_odd_metaphor", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="identify_odd_metaphor", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -implicatures = LightevalTaskConfig( - name="bigbench:implicatures", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="implicatures", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -implicit_relations = LightevalTaskConfig( - name="bigbench:implicit_relations", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="implicit_relations", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -intent_recognition = LightevalTaskConfig( - name="bigbench:intent_recognition", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="intent_recognition", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -international_phonetic_alphabet_nli = LightevalTaskConfig( - name="bigbench:international_phonetic_alphabet_nli", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="international_phonetic_alphabet_nli", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -international_phonetic_alphabet_transliterate = LightevalTaskConfig( - name="bigbench:international_phonetic_alphabet_transliterate", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="international_phonetic_alphabet_transliterate", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.bleu, Metrics.rouge_t5, Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -intersect_geometry = LightevalTaskConfig( - name="bigbench:intersect_geometry", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="intersect_geometry", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -irony_identification = LightevalTaskConfig( - name="bigbench:irony_identification", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="irony_identification", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -kanji_ascii = LightevalTaskConfig( - name="bigbench:kanji_ascii", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="kanji_ascii", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -kannada = LightevalTaskConfig( - name="bigbench:kannada", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="kannada", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -key_value_maps = LightevalTaskConfig( - name="bigbench:key_value_maps", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="key_value_maps", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -known_unknowns = LightevalTaskConfig( - name="bigbench:known_unknowns", - prompt_function=bigbench_linefeed_before_whitespace_after_query_prompt, - hf_repo="tasksource/bigbench", - hf_subset="known_unknowns", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -language_games = LightevalTaskConfig( - name="bigbench:language_games", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="language_games", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.bleu, Metrics.rouge_t5, Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -language_identification = LightevalTaskConfig( - name="bigbench:language_identification", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="language_identification", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -linguistic_mappings = LightevalTaskConfig( - name="bigbench:linguistic_mappings", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="linguistic_mappings", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -linguistics_puzzles = LightevalTaskConfig( - name="bigbench:linguistics_puzzles", - prompt_function=bigbench_whitespace_after_query_prompt, - hf_repo="tasksource/bigbench", - hf_subset="linguistics_puzzles", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.bleu, Metrics.rouge_t5, Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=None, - version=0, -) - -logic_grid_puzzle = LightevalTaskConfig( - name="bigbench:logic_grid_puzzle", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="logic_grid_puzzle", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -logical_args = LightevalTaskConfig( - name="bigbench:logical_args", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="logical_args", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -logical_deduction = LightevalTaskConfig( - name="bigbench:logical_deduction", - prompt_function=bigbench_whitespace_after_query_prompt, - hf_repo="tasksource/bigbench", - hf_subset="logical_deduction", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -logical_fallacy_detection = LightevalTaskConfig( - name="bigbench:logical_fallacy_detection", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="logical_fallacy_detection", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -logical_sequence = LightevalTaskConfig( - name="bigbench:logical_sequence", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="logical_sequence", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -mathematical_induction = LightevalTaskConfig( - name="bigbench:mathematical_induction", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="mathematical_induction", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -matrixshapes = LightevalTaskConfig( - name="bigbench:matrixshapes", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="matrixshapes", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -metaphor_boolean = LightevalTaskConfig( - name="bigbench:metaphor_boolean", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="metaphor_boolean", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -metaphor_understanding = LightevalTaskConfig( - name="bigbench:metaphor_understanding", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="metaphor_understanding", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -minute_mysteries_qa = LightevalTaskConfig( - name="bigbench:minute_mysteries_qa", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="minute_mysteries_qa", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc, Metrics.rouge_t5], - stop_sequence=["\n"], - version=0, -) - -misconceptions = LightevalTaskConfig( - name="bigbench:misconceptions", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="misconceptions", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -misconceptions_russian = LightevalTaskConfig( - name="bigbench:misconceptions_russian", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="misconceptions_russian", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -mnist_ascii = LightevalTaskConfig( - name="bigbench:mnist_ascii", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="mnist_ascii", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -modified_arithmetic = LightevalTaskConfig( - name="bigbench:modified_arithmetic", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="modified_arithmetic", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -moral_permissibility = LightevalTaskConfig( - name="bigbench:moral_permissibility", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="moral_permissibility", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -movie_dialog_same_or_different = LightevalTaskConfig( - name="bigbench:movie_dialog_same_or_different", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="movie_dialog_same_or_different", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -movie_recommendation = LightevalTaskConfig( - name="bigbench:movie_recommendation", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="movie_recommendation", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -mult_data_wrangling = LightevalTaskConfig( - name="bigbench:mult_data_wrangling", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="mult_data_wrangling", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -navigate = LightevalTaskConfig( - name="bigbench:navigate", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="navigate", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -nonsense_words_grammar = LightevalTaskConfig( - name="bigbench:nonsense_words_grammar", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="nonsense_words_grammar", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -novel_concepts = LightevalTaskConfig( - name="bigbench:novel_concepts", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="novel_concepts", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -object_counting = LightevalTaskConfig( - name="bigbench:object_counting", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="object_counting", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -odd_one_out = LightevalTaskConfig( - name="bigbench:odd_one_out", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="odd_one_out", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -operators = LightevalTaskConfig( - name="bigbench:operators", - prompt_function=bigbench_whitespace_after_query_prompt, - hf_repo="tasksource/bigbench", - hf_subset="operators", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -paragraph_segmentation = LightevalTaskConfig( - name="bigbench:paragraph_segmentation", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="paragraph_segmentation", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -parsinlu_qa = LightevalTaskConfig( - name="bigbench:parsinlu_qa", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="parsinlu_qa", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -parsinlu_reading_comprehension = LightevalTaskConfig( - name="bigbench:parsinlu_reading_comprehension", - prompt_function=bigbench_linefeed_before_whitespace_after_query_prompt, - hf_repo="tasksource/bigbench", - hf_subset="parsinlu_reading_comprehension", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=None, - version=0, -) - -penguins_in_a_table = LightevalTaskConfig( - name="bigbench:penguins_in_a_table", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="penguins_in_a_table", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc, Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -periodic_elements = LightevalTaskConfig( - name="bigbench:periodic_elements", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="periodic_elements", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -persian_idioms = LightevalTaskConfig( - name="bigbench:persian_idioms", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="persian_idioms", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -phrase_relatedness = LightevalTaskConfig( - name="bigbench:phrase_relatedness", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="phrase_relatedness", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -physical_intuition = LightevalTaskConfig( - name="bigbench:physical_intuition", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="physical_intuition", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -physics = LightevalTaskConfig( - name="bigbench:physics", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="physics", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -physics_questions = LightevalTaskConfig( - name="bigbench:physics_questions", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="physics_questions", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.bleu, Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -play_dialog_same_or_different = LightevalTaskConfig( - name="bigbench:play_dialog_same_or_different", - prompt_function=bigbench_linefeed_before_whitespace_after_query_prompt, - hf_repo="tasksource/bigbench", - hf_subset="play_dialog_same_or_different", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -polish_sequence_labeling = LightevalTaskConfig( - name="bigbench:polish_sequence_labeling", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="polish_sequence_labeling", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.f1_score], - stop_sequence=["\n"], - version=0, -) - -presuppositions_as_nli = LightevalTaskConfig( - name="bigbench:presuppositions_as_nli", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="presuppositions_as_nli", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -qa_wikidata = LightevalTaskConfig( - name="bigbench:qa_wikidata", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="qa_wikidata", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[ - Metrics.bleurt, - Metrics.bleu, - Metrics.rouge_t5, - Metrics.exact_match(sample_params={"strip_strings": False}), - ], - stop_sequence=["\n"], - version=0, -) - -question_selection = LightevalTaskConfig( - name="bigbench:question_selection", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="question_selection", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -real_or_fake_text = LightevalTaskConfig( - name="bigbench:real_or_fake_text", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="real_or_fake_text", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -reasoning_about_colored_objects = LightevalTaskConfig( - name="bigbench:reasoning_about_colored_objects", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="reasoning_about_colored_objects", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -repeat_copy_logic = LightevalTaskConfig( - name="bigbench:repeat_copy_logic", - prompt_function=bigbench_whitespace_after_query_prompt, - hf_repo="tasksource/bigbench", - hf_subset="repeat_copy_logic", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -rephrase = LightevalTaskConfig( - name="bigbench:rephrase", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="rephrase", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[ - Metrics.rouge_t5, - Metrics.bleu, - Metrics.loglikelihood_acc, - Metrics.exact_match(sample_params={"strip_strings": False}), - ], - stop_sequence=["\n"], - version=0, -) - -rhyming = LightevalTaskConfig( - name="bigbench:rhyming", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="rhyming", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -riddle_sense = LightevalTaskConfig( - name="bigbench:riddle_sense", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="riddle_sense", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -ruin_names = LightevalTaskConfig( - name="bigbench:ruin_names", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="ruin_names", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -salient_translation_error_detection = LightevalTaskConfig( - name="bigbench:salient_translation_error_detection", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="salient_translation_error_detection", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -scientific_press_release = LightevalTaskConfig( - name="bigbench:scientific_press_release", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="scientific_press_release", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.bleu, Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -semantic_parsing_in_context_sparc = LightevalTaskConfig( - name="bigbench:semantic_parsing_in_context_sparc", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="semantic_parsing_in_context_sparc", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.bleu, Metrics.rouge_t5, Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -semantic_parsing_spider = LightevalTaskConfig( - name="bigbench:semantic_parsing_spider", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="semantic_parsing_spider", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.bleu, Metrics.rouge_t5, Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -sentence_ambiguity = LightevalTaskConfig( - name="bigbench:sentence_ambiguity", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="sentence_ambiguity", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -similarities_abstraction = LightevalTaskConfig( - name="bigbench:similarities_abstraction", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="similarities_abstraction", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.bleu, Metrics.rouge_t5, Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -simp_turing_concept = LightevalTaskConfig( - name="bigbench:simp_turing_concept", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="simp_turing_concept", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -simple_arithmetic_json = LightevalTaskConfig( - name="bigbench:simple_arithmetic_json", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="simple_arithmetic_json", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -simple_arithmetic_json_multiple_choice = LightevalTaskConfig( - name="bigbench:simple_arithmetic_json_multiple_choice", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="simple_arithmetic_json_multiple_choice", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -simple_arithmetic_json_subtasks = LightevalTaskConfig( - name="bigbench:simple_arithmetic_json_subtasks", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="simple_arithmetic_json_subtasks", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -simple_arithmetic_multiple_targets_json = LightevalTaskConfig( - name="bigbench:simple_arithmetic_multiple_targets_json", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="simple_arithmetic_multiple_targets_json", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.bleu, Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -simple_ethical_questions = LightevalTaskConfig( - name="bigbench:simple_ethical_questions", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="simple_ethical_questions", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -simple_text_editing = LightevalTaskConfig( - name="bigbench:simple_text_editing", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="simple_text_editing", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -snarks = LightevalTaskConfig( - name="bigbench:snarks", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="snarks", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -social_iqa = LightevalTaskConfig( - name="bigbench:social_iqa", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="social_iqa", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -social_support = LightevalTaskConfig( - name="bigbench:social_support", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="social_support", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.f1_score_macro], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -sports_understanding = LightevalTaskConfig( - name="bigbench:sports_understanding", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="sports_understanding", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -strange_stories = LightevalTaskConfig( - name="bigbench:strange_stories", - prompt_function=bigbench_whitespace_after_query_prompt, - hf_repo="tasksource/bigbench", - hf_subset="strange_stories", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -strategyqa = LightevalTaskConfig( - name="bigbench:strategyqa", - prompt_function=bigbench_linefeed_before_whitespace_after_query_prompt, - hf_repo="tasksource/bigbench", - hf_subset="strategyqa", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.bleu, Metrics.rouge_t5, Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -sufficient_information = LightevalTaskConfig( - name="bigbench:sufficient_information", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="sufficient_information", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -suicide_risk = LightevalTaskConfig( - name="bigbench:suicide_risk", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="suicide_risk", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -swahili_english_proverbs = LightevalTaskConfig( - name="bigbench:swahili_english_proverbs", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="swahili_english_proverbs", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -swedish_to_german_proverbs = LightevalTaskConfig( - name="bigbench:swedish_to_german_proverbs", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="swedish_to_german_proverbs", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -symbol_interpretation = LightevalTaskConfig( - name="bigbench:symbol_interpretation", - prompt_function=bigbench_linefeed_before_whitespace_after_query_prompt, - hf_repo="tasksource/bigbench", - hf_subset="symbol_interpretation", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -tellmewhy = LightevalTaskConfig( - name="bigbench:tellmewhy", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="tellmewhy", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.bleu, Metrics.rouge_t5], - stop_sequence=["\n"], - version=0, -) - -temporal_sequences = LightevalTaskConfig( - name="bigbench:temporal_sequences", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="temporal_sequences", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -tense = LightevalTaskConfig( - name="bigbench:tense", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="tense", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -timedial = LightevalTaskConfig( - name="bigbench:timedial", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="timedial", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -topical_chat = LightevalTaskConfig( - name="bigbench:topical_chat", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="topical_chat", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.bleu, Metrics.rouge_t5, Metrics.loglikelihood_acc, Metrics.bleurt], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -tracking_shuffled_objects = LightevalTaskConfig( - name="bigbench:tracking_shuffled_objects", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="tracking_shuffled_objects", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -understanding_fables = LightevalTaskConfig( - name="bigbench:understanding_fables", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="understanding_fables", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -undo_permutation = LightevalTaskConfig( - name="bigbench:undo_permutation", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="undo_permutation", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -unit_conversion = LightevalTaskConfig( - name="bigbench:unit_conversion", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="unit_conversion", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -unit_interpretation = LightevalTaskConfig( - name="bigbench:unit_interpretation", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="unit_interpretation", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -unnatural_in_context_learning = LightevalTaskConfig( - name="bigbench:unnatural_in_context_learning", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="unnatural_in_context_learning", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -vitaminc_fact_verification = LightevalTaskConfig( - name="bigbench:vitaminc_fact_verification", - prompt_function=bigbench_whitespace_after_query_prompt, - hf_repo="tasksource/bigbench", - hf_subset="vitaminc_fact_verification", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -what_is_the_tao = LightevalTaskConfig( - name="bigbench:what_is_the_tao", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="what_is_the_tao", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -which_wiki_edit = LightevalTaskConfig( - name="bigbench:which_wiki_edit", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="which_wiki_edit", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -winowhy = LightevalTaskConfig( - name="bigbench:winowhy", - prompt_function=bigbench_whitespace_after_query_prompt, - hf_repo="tasksource/bigbench", - hf_subset="winowhy", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -word_sorting = LightevalTaskConfig( - name="bigbench:word_sorting", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="word_sorting", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -word_unscrambling = LightevalTaskConfig( - name="bigbench:word_unscrambling", - prompt_function=bigbench_prompt, - hf_repo="tasksource/bigbench", - hf_subset="word_unscrambling", - hf_avail_splits=["default", "train", "validation"], - evaluation_splits=["default"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - abstract_narrative_understanding, - anachronisms, - analogical_similarity, - moral_permissibility, - movie_dialog_same_or_different, - movie_recommendation, - mult_data_wrangling, - simple_ethical_questions, - simple_text_editing, - snarks, - social_iqa, - social_support, - sports_understanding, - strange_stories, - strategyqa, - sufficient_information, - suicide_risk, - swahili_english_proverbs, - swedish_to_german_proverbs, - symbol_interpretation, - tellmewhy, - temporal_sequences, - tense, - timedial, - topical_chat, - tracking_shuffled_objects, - understanding_fables, - undo_permutation, - unit_conversion, - unit_interpretation, - unnatural_in_context_learning, - vitaminc_fact_verification, - what_is_the_tao, - which_wiki_edit, - winowhy, - word_sorting, - word_unscrambling, -] diff --git a/src/lighteval/tasks/tasks/bigbench_extra_hard.py b/src/lighteval/tasks/tasks/bigbench_extra_hard.py deleted file mode 100644 index f1dfc060a..000000000 --- a/src/lighteval/tasks/tasks/bigbench_extra_hard.py +++ /dev/null @@ -1,241 +0,0 @@ -""" -name: -BIG-Bench Extra Hard - -dataset: -jgyasu/bbeh - -abstract: -BIG-Bench Extra Hard (BBEH) is a successor to BIG-Bench Hard (BBH), created to evaluate large -language models on substantially more difficult general-reasoning tasks. Each BBH task is replaced -with a new task targeting the same underlying reasoning skill but at a significantly higher difficulty. - -languages: -english - -tags: -reasoning - -paper: -https://arxiv.org/abs/2502.19187 - -starred: -true -""" - -from inspect_ai.dataset import Sample -from inspect_ai.scorer import answer -from inspect_ai.solver import generate, system_message - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def bbeh_prompt(line, task_name: str = None): - line = {k: v for k, v in line.items() if v is not None} - - query = "Question: \n" - query += line["input"] - query += "\nAnswer:" - - return Doc( - task_name=task_name, - query=query, - choices=line["target"], - gold_index=0, - instruction="", - ) - - -def record_to_sample(record): - query = f"{record['input']}" - target = record["target"] - - return Sample(input=query, target=target) - - -SYSTEM_MESSAGE = """Submit your answer in the following format: -ANSWER: {your answer} -""" - - -COMMON_TASK_ARGS = { - "prompt_function": bbeh_prompt, - "hf_repo": "jgyasu/bbeh", - "hf_avail_splits": ["train"], - "evaluation_splits": ["train"], - "few_shots_split": None, - "few_shots_select": None, - "generation_size": -1, - "metrics": [Metrics.loglikelihood_acc], - "stop_sequence": ["", "Q=", "\n\n"], - "version": 0, - "sample_fields": record_to_sample, - "solver": [system_message(SYSTEM_MESSAGE), generate(cache=True)], - "scorer": answer(pattern="line"), -} - -boardgame_qa = LightevalTaskConfig( - name="bigbench_extra_hard:boardgame_qa", - hf_subset="boardgame_qa", - **COMMON_TASK_ARGS, -) - -boolean_expressions = LightevalTaskConfig( - name="bigbench_extra_hard:boolean_expressions", - hf_subset="boolean_expressions", - **COMMON_TASK_ARGS, -) - -buggy_tables = LightevalTaskConfig( - name="bigbench_extra_hard:buggy_tables", - hf_subset="buggy_tables", - **COMMON_TASK_ARGS, -) - -causal_understanding = LightevalTaskConfig( - name="bigbench_extra_hard:causal_understanding", - hf_subset="causal_understanding", - **COMMON_TASK_ARGS, -) - -disambiguation_qa = LightevalTaskConfig( - name="bigbench_extra_hard:disambiguation_qa", - hf_subset="disambiguation_qa", - **COMMON_TASK_ARGS, -) - -dyck_languages = LightevalTaskConfig( - name="bigbench_extra_hard:dyck_languages", - hf_subset="dyck_languages", - **COMMON_TASK_ARGS, -) - -geometric_shapes = LightevalTaskConfig( - name="bigbench_extra_hard:geometric_shapes", - hf_subset="geometric_shapes", - **COMMON_TASK_ARGS, -) - -hyperbaton = LightevalTaskConfig( - name="bigbench_extra_hard:hyperbaton", - hf_subset="hyperbaton", - **COMMON_TASK_ARGS, -) - -linguini = LightevalTaskConfig( - name="bigbench_extra_hard:linguini", - hf_subset="linguini", - **COMMON_TASK_ARGS, -) - -movie_recommendation = LightevalTaskConfig( - name="bigbench_extra_hard:movie_recommendation", - hf_subset="movie_recommendation", - **COMMON_TASK_ARGS, -) - -multistep_arithmetic = LightevalTaskConfig( - name="bigbench_extra_hard:multistep_arithmetic", - hf_subset="multistep_arithmetic", - **COMMON_TASK_ARGS, -) - -nycc = LightevalTaskConfig( - name="bigbench_extra_hard:nycc", - hf_subset="nycc", - **COMMON_TASK_ARGS, -) - -object_counting = LightevalTaskConfig( - name="bigbench_extra_hard:object_counting", - hf_subset="object_counting", - **COMMON_TASK_ARGS, -) - -object_properties = LightevalTaskConfig( - name="bigbench_extra_hard:object_properties", - hf_subset="object_properties", - **COMMON_TASK_ARGS, -) - -sarc_triples = LightevalTaskConfig( - name="bigbench_extra_hard:sarc_triples", - hf_subset="sarc_triples", - **COMMON_TASK_ARGS, -) - -shuffled_objects = LightevalTaskConfig( - name="bigbench_extra_hard:shuffled_objects", - hf_subset="shuffled_objects", - **COMMON_TASK_ARGS, -) - -spatial_reasoning = LightevalTaskConfig( - name="bigbench_extra_hard:spatial_reasoning", - hf_subset="spatial_reasoning", - **COMMON_TASK_ARGS, -) - -sportqa = LightevalTaskConfig( - name="bigbench_extra_hard:sportqa", - hf_subset="sportqa", - **COMMON_TASK_ARGS, -) - -temporal_sequence = LightevalTaskConfig( - name="bigbench_extra_hard:temporal_sequence", - hf_subset="temporal_sequence", - **COMMON_TASK_ARGS, -) - -time_arithmetic = LightevalTaskConfig( - name="bigbench_extra_hard:time_arithmetic", - hf_subset="time_arithmetic", - **COMMON_TASK_ARGS, -) - -web_of_lies = LightevalTaskConfig( - name="bigbench_extra_hard:web_of_lies", - hf_subset="web_of_lies", - **COMMON_TASK_ARGS, -) - -word_sorting = LightevalTaskConfig( - name="bigbench_extra_hard:word_sorting", - hf_subset="word_sorting", - **COMMON_TASK_ARGS, -) - -zebra_puzzles = LightevalTaskConfig( - name="bigbench_extra_hard:zebra_puzzles", - hf_subset="zebra_puzzles", - **COMMON_TASK_ARGS, -) - -TASKS_TABLE = [ - boardgame_qa, - boolean_expressions, - buggy_tables, - causal_understanding, - disambiguation_qa, - dyck_languages, - geometric_shapes, - hyperbaton, - linguini, - movie_recommendation, - multistep_arithmetic, - nycc, - object_counting, - object_properties, - sarc_triples, - shuffled_objects, - spatial_reasoning, - sportqa, - temporal_sequence, - time_arithmetic, - web_of_lies, - word_sorting, - zebra_puzzles, -] diff --git a/src/lighteval/tasks/tasks/bigbench_hard.py b/src/lighteval/tasks/tasks/bigbench_hard.py deleted file mode 100644 index 18ebef8c0..000000000 --- a/src/lighteval/tasks/tasks/bigbench_hard.py +++ /dev/null @@ -1,399 +0,0 @@ -""" -name: -Bigbench Hard - -dataset: -lighteval/bbh - -abstract: - -languages: - -tags: -reasoning - -paper: -""" - -from string import ascii_uppercase - -from inspect_ai.dataset import Sample -from inspect_ai.scorer import choice -from inspect_ai.solver import multiple_choice - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def bbh_prompt(line, task_name: str = None): - line = {k: v for k, v in line.items() if v is not None} - - query = line.get("task_prefix", "") - query += line.get("example_input_prefix", "\nQuestion: ") - query += line["input"] - query += line.get("choice_prefix", "\n Choices: ") - query += "".join([f"\n{key}. {choice}" for key, choice in zip(ascii_uppercase, line["choices"])]) - query += line.get("example_output_prefix", "\nAnswer: ") - - return Doc( - task_name=task_name, - query=query, - choices=list(ascii_uppercase[: len(line["choices"])]), - gold_index=line["target_idx"], - instruction=line.get("task_prefix", None), - ) - - -def record_to_sample(record): - query = f"{record.get('task_prefix', '')}\n{record['input']}" - target = ascii_uppercase[record["target_idx"]] - choices = record["choices"] - - return Sample(input=query, target=target, choices=choices) - - -causal_judgment = LightevalTaskConfig( - name="bigbench_hard:causal_judgment", - prompt_function=bbh_prompt, - hf_repo="lighteval/bbh", - hf_subset="causal_judgement", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["", "Q=", "\n\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -date_understanding = LightevalTaskConfig( - name="bigbench_hard:date_understanding", - prompt_function=bbh_prompt, - hf_repo="lighteval/bbh", - hf_subset="date_understanding", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["", "Q=", "\n\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -disambiguation_qa = LightevalTaskConfig( - name="bigbench_hard:disambiguation_qa", - prompt_function=bbh_prompt, - hf_repo="lighteval/bbh", - hf_subset="disambiguation_qa", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["", "Q=", "\n\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -geometric_shapes = LightevalTaskConfig( - name="bigbench_hard:geometric_shapes", - prompt_function=bbh_prompt, - hf_repo="lighteval/bbh", - hf_subset="geometric_shapes", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["", "Q=", "\n\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -logical_deduction_five_objects = LightevalTaskConfig( - name="bigbench_hard:logical_deduction_five_objects", - prompt_function=bbh_prompt, - hf_repo="lighteval/bbh", - hf_subset="logical_deduction_five_objects", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["", "Q=", "\n\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -logical_deduction_seven_objects = LightevalTaskConfig( - name="bigbench_hard:logical_deduction_seven_objects", - prompt_function=bbh_prompt, - hf_repo="lighteval/bbh", - hf_subset="logical_deduction_seven_objects", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["", "Q=", "\n\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -logical_deduction_three_objects = LightevalTaskConfig( - name="bigbench_hard:logical_deduction_three_objects", - prompt_function=bbh_prompt, - hf_repo="lighteval/bbh", - hf_subset="logical_deduction_three_objects", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["", "Q=", "\n\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -movie_recommendation = LightevalTaskConfig( - name="bigbench_hard:movie_recommendation", - prompt_function=bbh_prompt, - hf_repo="lighteval/bbh", - hf_subset="movie_recommendation", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["", "Q=", "\n\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -navigate = LightevalTaskConfig( - name="bigbench_hard:navigate", - prompt_function=bbh_prompt, - hf_repo="lighteval/bbh", - hf_subset="navigate", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["", "Q=", "\n\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -reasoning_about_colored_objects = LightevalTaskConfig( - name="bigbench_hard:reasoning_about_colored_objects", - prompt_function=bbh_prompt, - hf_repo="lighteval/bbh", - hf_subset="reasoning_about_colored_objects", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["", "Q=", "\n\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -ruin_names = LightevalTaskConfig( - name="bigbench_hard:ruin_names", - prompt_function=bbh_prompt, - hf_repo="lighteval/bbh", - hf_subset="ruin_names", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["", "Q=", "\n\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -salient_translation_error_detection = LightevalTaskConfig( - name="bigbench_hard:salient_translation_error_detection", - prompt_function=bbh_prompt, - hf_repo="lighteval/bbh", - hf_subset="salient_translation_error_detection", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["", "Q=", "\n\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -snarks = LightevalTaskConfig( - name="bigbench_hard:snarks", - prompt_function=bbh_prompt, - hf_repo="lighteval/bbh", - hf_subset="snarks", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["", "Q=", "\n\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -sports_understanding = LightevalTaskConfig( - name="bigbench_hard:sports_understanding", - prompt_function=bbh_prompt, - hf_repo="lighteval/bbh", - hf_subset="sports_understanding", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["", "Q=", "\n\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -temporal_sequences = LightevalTaskConfig( - name="bigbench_hard:temporal_sequences", - prompt_function=bbh_prompt, - hf_repo="lighteval/bbh", - hf_subset="temporal_sequences", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["", "Q=", "\n\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -tracking_shuffled_objects_five_objects = LightevalTaskConfig( - name="bigbench_hard:tracking_shuffled_objects_five_objects", - prompt_function=bbh_prompt, - hf_repo="lighteval/bbh", - hf_subset="tracking_shuffled_objects_five_objects", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["", "Q=", "\n\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -tracking_shuffled_objects_seven_objects = LightevalTaskConfig( - name="bigbench_hard:tracking_shuffled_objects_seven_objects", - prompt_function=bbh_prompt, - hf_repo="lighteval/bbh", - hf_subset="tracking_shuffled_objects_seven_objects", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["", "Q=", "\n\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -tracking_shuffled_objects_three_objects = LightevalTaskConfig( - name="bigbench_hard:tracking_shuffled_objects_three_objects", - prompt_function=bbh_prompt, - hf_repo="lighteval/bbh", - hf_subset="tracking_shuffled_objects_three_objects", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["", "Q=", "\n\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -TASKS_TABLE = [ - causal_judgment, - date_understanding, - disambiguation_qa, - geometric_shapes, - logical_deduction_five_objects, - logical_deduction_seven_objects, - logical_deduction_three_objects, - movie_recommendation, - navigate, - reasoning_about_colored_objects, - ruin_names, - salient_translation_error_detection, - snarks, - sports_understanding, - temporal_sequences, - tracking_shuffled_objects_five_objects, - tracking_shuffled_objects_seven_objects, - tracking_shuffled_objects_three_objects, -] diff --git a/src/lighteval/tasks/tasks/blimp.py b/src/lighteval/tasks/tasks/blimp.py deleted file mode 100644 index 08dc09d65..000000000 --- a/src/lighteval/tasks/tasks/blimp.py +++ /dev/null @@ -1,1081 +0,0 @@ -""" -name: -Blimp - -dataset: -nyu-mll/blimp - -abstract: -BLiMP is a challenge set for evaluating what language models (LMs) know -about major grammatical phenomena in English. BLiMP consists of 67 -sub-datasets, each containing 1000 minimal pairs isolating specific -contrasts in syntax, morphology, or semantics. The data is automatically -generated according to expert-crafted grammars. - -languages: -english - -tags: -language-modeling - -paper: -https://arxiv.org/abs/1912.00582 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -# TODO: Convert to inspect-ai - - -def blimp_prompt(line, task_name: str = None): - return Doc(task_name=task_name, query="", choices=[line["sentence_good"], line["sentence_bad"]], gold_index=0) - - -blimp_adjunct_island = LightevalTaskConfig( - name="blimp:adjunct_island", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="adjunct_island", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_anaphor_gender_agreement = LightevalTaskConfig( - name="blimp:anaphor_gender_agreement", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="anaphor_gender_agreement", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_anaphor_number_agreement = LightevalTaskConfig( - name="blimp:anaphor_number_agreement", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="anaphor_number_agreement", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_animate_subject_passive = LightevalTaskConfig( - name="blimp:animate_subject_passive", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="animate_subject_passive", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_animate_subject_trans = LightevalTaskConfig( - name="blimp:animate_subject_trans", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="animate_subject_trans", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_causative = LightevalTaskConfig( - name="blimp:causative", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="causative", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_complex_NP_island = LightevalTaskConfig( - name="blimp:complex_NP_island", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="complex_NP_island", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_coordinate_structure_constraint_complex_left_branch = LightevalTaskConfig( - name="blimp:coordinate_structure_constraint_complex_left_branch", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="coordinate_structure_constraint_complex_left_branch", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_coordinate_structure_constraint_object_extraction = LightevalTaskConfig( - name="blimp:coordinate_structure_constraint_object_extraction", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="coordinate_structure_constraint_object_extraction", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_determiner_noun_agreement_1 = LightevalTaskConfig( - name="blimp:determiner_noun_agreement_1", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="determiner_noun_agreement_1", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_determiner_noun_agreement_2 = LightevalTaskConfig( - name="blimp:determiner_noun_agreement_2", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="determiner_noun_agreement_2", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_determiner_noun_agreement_irregular_1 = LightevalTaskConfig( - name="blimp:determiner_noun_agreement_irregular_1", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="determiner_noun_agreement_irregular_1", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_determiner_noun_agreement_irregular_2 = LightevalTaskConfig( - name="blimp:determiner_noun_agreement_irregular_2", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="determiner_noun_agreement_irregular_2", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_determiner_noun_agreement_with_adj_2 = LightevalTaskConfig( - name="blimp:determiner_noun_agreement_with_adj_2", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="determiner_noun_agreement_with_adj_2", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_determiner_noun_agreement_with_adj_irregular_1 = LightevalTaskConfig( - name="blimp:determiner_noun_agreement_with_adj_irregular_1", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="determiner_noun_agreement_with_adj_irregular_1", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_determiner_noun_agreement_with_adj_irregular_2 = LightevalTaskConfig( - name="blimp:determiner_noun_agreement_with_adj_irregular_2", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="determiner_noun_agreement_with_adj_irregular_2", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_determiner_noun_agreement_with_adjective_1 = LightevalTaskConfig( - name="blimp:determiner_noun_agreement_with_adjective_1", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="determiner_noun_agreement_with_adjective_1", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_distractor_agreement_relational_noun = LightevalTaskConfig( - name="blimp:distractor_agreement_relational_noun", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="distractor_agreement_relational_noun", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_distractor_agreement_relative_clause = LightevalTaskConfig( - name="blimp:distractor_agreement_relative_clause", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="distractor_agreement_relative_clause", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_drop_argument = LightevalTaskConfig( - name="blimp:drop_argument", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="drop_argument", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_ellipsis_n_bar_1 = LightevalTaskConfig( - name="blimp:ellipsis_n_bar_1", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="ellipsis_n_bar_1", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_ellipsis_n_bar_2 = LightevalTaskConfig( - name="blimp:ellipsis_n_bar_2", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="ellipsis_n_bar_2", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_existential_there_object_raising = LightevalTaskConfig( - name="blimp:existential_there_object_raising", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="existential_there_object_raising", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_existential_there_quantifiers_1 = LightevalTaskConfig( - name="blimp:existential_there_quantifiers_1", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="existential_there_quantifiers_1", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_existential_there_quantifiers_2 = LightevalTaskConfig( - name="blimp:existential_there_quantifiers_2", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="existential_there_quantifiers_2", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_existential_there_subject_raising = LightevalTaskConfig( - name="blimp:existential_there_subject_raising", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="existential_there_subject_raising", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_expletive_it_object_raising = LightevalTaskConfig( - name="blimp:expletive_it_object_raising", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="expletive_it_object_raising", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_inchoative = LightevalTaskConfig( - name="blimp:inchoative", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="inchoative", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_intransitive = LightevalTaskConfig( - name="blimp:intransitive", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="intransitive", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_irregular_past_participle_adjectives = LightevalTaskConfig( - name="blimp:irregular_past_participle_adjectives", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="irregular_past_participle_adjectives", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_irregular_past_participle_verbs = LightevalTaskConfig( - name="blimp:irregular_past_participle_verbs", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="irregular_past_participle_verbs", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_irregular_plural_subject_verb_agreement_1 = LightevalTaskConfig( - name="blimp:irregular_plural_subject_verb_agreement_1", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="irregular_plural_subject_verb_agreement_1", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_irregular_plural_subject_verb_agreement_2 = LightevalTaskConfig( - name="blimp:irregular_plural_subject_verb_agreement_2", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="irregular_plural_subject_verb_agreement_2", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_left_branch_island_echo_question = LightevalTaskConfig( - name="blimp:left_branch_island_echo_question", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="left_branch_island_echo_question", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_left_branch_island_simple_question = LightevalTaskConfig( - name="blimp:left_branch_island_simple_question", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="left_branch_island_simple_question", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_matrix_question_npi_licensor_present = LightevalTaskConfig( - name="blimp:matrix_question_npi_licensor_present", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="matrix_question_npi_licensor_present", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_npi_present_1 = LightevalTaskConfig( - name="blimp:npi_present_1", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="npi_present_1", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_npi_present_2 = LightevalTaskConfig( - name="blimp:npi_present_2", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="npi_present_2", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_only_npi_licensor_present = LightevalTaskConfig( - name="blimp:only_npi_licensor_present", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="only_npi_licensor_present", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_only_npi_scope = LightevalTaskConfig( - name="blimp:only_npi_scope", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="only_npi_scope", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_passive_1 = LightevalTaskConfig( - name="blimp:passive_1", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="passive_1", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_passive_2 = LightevalTaskConfig( - name="blimp:passive_2", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="passive_2", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_principle_A_c_command = LightevalTaskConfig( - name="blimp:principle_A_c_command", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="principle_A_c_command", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_principle_A_case_1 = LightevalTaskConfig( - name="blimp:principle_A_case_1", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="principle_A_case_1", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_principle_A_case_2 = LightevalTaskConfig( - name="blimp:principle_A_case_2", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="principle_A_case_2", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_principle_A_domain_1 = LightevalTaskConfig( - name="blimp:principle_A_domain_1", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="principle_A_domain_1", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_principle_A_domain_2 = LightevalTaskConfig( - name="blimp:principle_A_domain_2", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="principle_A_domain_2", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_principle_A_domain_3 = LightevalTaskConfig( - name="blimp:principle_A_domain_3", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="principle_A_domain_3", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_principle_A_reconstruction = LightevalTaskConfig( - name="blimp:principle_A_reconstruction", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="principle_A_reconstruction", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_regular_plural_subject_verb_agreement_1 = LightevalTaskConfig( - name="blimp:regular_plural_subject_verb_agreement_1", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="regular_plural_subject_verb_agreement_1", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_regular_plural_subject_verb_agreement_2 = LightevalTaskConfig( - name="blimp:regular_plural_subject_verb_agreement_2", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="regular_plural_subject_verb_agreement_2", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_sentential_negation_npi_licensor_present = LightevalTaskConfig( - name="blimp:sentential_negation_npi_licensor_present", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="sentential_negation_npi_licensor_present", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_sentential_negation_npi_scope = LightevalTaskConfig( - name="blimp:sentential_negation_npi_scope", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="sentential_negation_npi_scope", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_sentential_subject_island = LightevalTaskConfig( - name="blimp:sentential_subject_island", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="sentential_subject_island", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_superlative_quantifiers_1 = LightevalTaskConfig( - name="blimp:superlative_quantifiers_1", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="superlative_quantifiers_1", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_superlative_quantifiers_2 = LightevalTaskConfig( - name="blimp:superlative_quantifiers_2", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="superlative_quantifiers_2", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_tough_vs_raising_1 = LightevalTaskConfig( - name="blimp:tough_vs_raising_1", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="tough_vs_raising_1", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_tough_vs_raising_2 = LightevalTaskConfig( - name="blimp:tough_vs_raising_2", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="tough_vs_raising_2", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_transitive = LightevalTaskConfig( - name="blimp:transitive", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="transitive", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_wh_island = LightevalTaskConfig( - name="blimp:wh_island", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="wh_island", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_wh_questions_object_gap = LightevalTaskConfig( - name="blimp:wh_questions_object_gap", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="wh_questions_object_gap", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_wh_questions_subject_gap = LightevalTaskConfig( - name="blimp:wh_questions_subject_gap", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="wh_questions_subject_gap", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_wh_questions_subject_gap_long_distance = LightevalTaskConfig( - name="blimp:wh_questions_subject_gap_long_distance", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="wh_questions_subject_gap_long_distance", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_wh_vs_that_no_gap = LightevalTaskConfig( - name="blimp:wh_vs_that_no_gap", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="wh_vs_that_no_gap", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_wh_vs_that_no_gap_long_distance = LightevalTaskConfig( - name="blimp:wh_vs_that_no_gap_long_distance", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="wh_vs_that_no_gap_long_distance", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_wh_vs_that_with_gap = LightevalTaskConfig( - name="blimp:wh_vs_that_with_gap", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="wh_vs_that_with_gap", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -blimp_wh_vs_that_with_gap_long_distance = LightevalTaskConfig( - name="blimp:wh_vs_that_with_gap_long_distance", - prompt_function=blimp_prompt, - hf_repo="nyu-mll/blimp", - hf_subset="wh_vs_that_with_gap_long_distance", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - blimp_adjunct_island, - blimp_anaphor_gender_agreement, - blimp_anaphor_number_agreement, - blimp_animate_subject_passive, - blimp_animate_subject_trans, - blimp_causative, - blimp_complex_NP_island, - blimp_drop_argument, - blimp_ellipsis_n_bar_1, - blimp_ellipsis_n_bar_2, - blimp_existential_there_object_raising, - blimp_inchoative, - blimp_intransitive, - blimp_irregular_past_participle_adjectives, - blimp_irregular_past_participle_verbs, - blimp_only_npi_scope, - blimp_passive_1, - blimp_passive_2, - blimp_principle_A_c_command, - blimp_principle_A_reconstruction, - blimp_regular_plural_subject_verb_agreement_1, - blimp_regular_plural_subject_verb_agreement_2, - blimp_sentential_negation_npi_licensor_present, - blimp_sentential_negation_npi_scope, - blimp_sentential_subject_island, - blimp_superlative_quantifiers_1, - blimp_superlative_quantifiers_2, - blimp_tough_vs_raising_1, - blimp_tough_vs_raising_2, - blimp_transitive, - blimp_wh_island, - blimp_wh_questions_object_gap, - blimp_wh_questions_subject_gap, - blimp_wh_questions_subject_gap_long_distance, - blimp_wh_vs_that_no_gap, - blimp_wh_vs_that_no_gap_long_distance, - blimp_wh_vs_that_with_gap, - blimp_wh_vs_that_with_gap_long_distance, -] diff --git a/src/lighteval/tasks/tasks/bold.py b/src/lighteval/tasks/tasks/bold.py deleted file mode 100644 index ad4dd7b15..000000000 --- a/src/lighteval/tasks/tasks/bold.py +++ /dev/null @@ -1,156 +0,0 @@ -""" -name: -Bold - -dataset: -lighteval/bold_helm - -abstract: -The Bias in Open-Ended Language Generation Dataset (BOLD) for measuring biases -and toxicity in open-ended language generation. - -languages: -english - -tags: -bias, generation - -paper: -https://dl.acm.org/doi/10.1145/3442188.3445924 -""" - -from inspect_ai.dataset import Sample -from inspect_ai.scorer import exact -from inspect_ai.solver import generate - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def bold_prompt(line, task_name: str = None): - return Doc(task_name=task_name, query=line["text"], choices=None, gold_index=None) - - -def record_to_sample(record): - query = record["text"] - target = "" - return Sample(input=query, target=target) - - -bold = LightevalTaskConfig( - name="bold", - prompt_function=bold_prompt, - hf_repo="lighteval/bold_helm", - hf_subset="all", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.prediction_perplexity], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=exact(), -) - -bold_gender = LightevalTaskConfig( - name="bold:gender", - prompt_function=bold_prompt, - hf_repo="lighteval/bold_helm", - hf_subset="gender", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.prediction_perplexity], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=exact(), -) - -bold_political_ideology = LightevalTaskConfig( - name="bold:political_ideology", - prompt_function=bold_prompt, - hf_repo="lighteval/bold_helm", - hf_subset="political_ideology", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.prediction_perplexity], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=exact(), -) - -bold_profession = LightevalTaskConfig( - name="bold:profession", - prompt_function=bold_prompt, - hf_repo="lighteval/bold_helm", - hf_subset="profession", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.prediction_perplexity], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=exact(), -) - -bold_race = LightevalTaskConfig( - name="bold:race", - prompt_function=bold_prompt, - hf_repo="lighteval/bold_helm", - hf_subset="race", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.prediction_perplexity], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=exact(), -) - -bold_religious_ideology = LightevalTaskConfig( - name="bold:religious_ideology", - prompt_function=bold_prompt, - hf_repo="lighteval/bold_helm", - hf_subset="religious_ideology", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.prediction_perplexity], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=exact(), -) - -TASKS_TABLE = [ - bold, - bold_gender, - bold_political_ideology, - bold_profession, - bold_race, - bold_religious_ideology, -] diff --git a/src/lighteval/tasks/tasks/boolq.py b/src/lighteval/tasks/tasks/boolq.py deleted file mode 100644 index f927f9491..000000000 --- a/src/lighteval/tasks/tasks/boolq.py +++ /dev/null @@ -1,119 +0,0 @@ -""" -name: -Boolq - -dataset: -lighteval/boolq_helm - -abstract: -The BoolQ benchmark for binary (yes/no) question answering. - -languages: -english - -tags: -qa - -paper: -https://arxiv.org/abs/1905.11946 -""" - -from string import ascii_uppercase - -from inspect_ai.dataset import Sample -from inspect_ai.scorer import choice -from inspect_ai.solver import multiple_choice - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def boolq_prompt(line, task_name: str = None): - question = line["question"][:-1] if line["question"][-2:] == "??" else line["question"] - return Doc( - task_name=task_name, - query=f"Passage: {line['passage']}\nQuestion: {question}\nAnswer:", - choices=[" Yes", " No"], - gold_index=["Yes", "No"].index(line["answer"]), - ) - - -def boolq_contrastset_prompt(line, task_name: str = None): - if line["contrast_inputs"] in [None, ""]: - return boolq_prompt(line) - - return [ - Doc( - task_name=task_name, - query=f"{passage}\nQuestion: {question}\nAnswer:", - choices=["Yes", "No"], - gold_index=["No", "Yes"].index(line["answer"]), - ) - for passage, question in zip(line["contrast_inputs"]["passage"], line["contrast_inputs"]["question"]) - ][0] - - -def record_to_sample(record): - choices = ["Yes", "No"] - query = f"{record['passage']}\n{record['question']}" - target = ascii_uppercase[choices.index(record["answer"])] - return Sample(input=query, target=target, choices=choices) - - -def record_to_sample_contrastset(record): - if record["contrast_inputs"] in [None, ""]: - return record_to_sample(record) - - choices = ["Yes", "No"] - query = f"{record['contrast_inputs']['passage']}\n{record['contrast_inputs']['question']}" - target = ascii_uppercase[choices.index(record["answer"])] - - return Sample(input=query, target=target, choices=choices) - - -boolq = LightevalTaskConfig( - name="boolq", - prompt_function=boolq_prompt, - hf_repo="lighteval/boolq_helm", - hf_subset="default", - hf_avail_splits=["train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - - -boolq_contrastset = LightevalTaskConfig( - name="boolq:contrastset", - prompt_function=boolq_contrastset_prompt, - hf_repo="lighteval/boolq_helm", - hf_subset="default", - hf_avail_splits=["validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample_contrastset, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -TASKS_TABLE = [ - boolq, - boolq_contrastset, -] diff --git a/src/lighteval/tasks/tasks/civil_comments.py b/src/lighteval/tasks/tasks/civil_comments.py deleted file mode 100644 index 6c8fe51ec..000000000 --- a/src/lighteval/tasks/tasks/civil_comments.py +++ /dev/null @@ -1,201 +0,0 @@ -""" -name: -Civil Comments - -dataset: -lighteval/civil_comments_helm - -abstract: -The CivilComments benchmark for toxicity detection. - -languages: -english - -tags: -bias, classification - -paper: -https://arxiv.org/abs/1903.04561 -""" - -from string import ascii_uppercase - -from inspect_ai.dataset import Sample -from inspect_ai.scorer import choice -from inspect_ai.solver import multiple_choice - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def civil_comments_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"Passage: {line['text']}\nIs the passage above toxic?\nAnswer: ", - choices=[str(line["gold"])], - gold_index=0, - ) - - -def record_to_sample(record): - query = f"{record['text']}\nIs the passage above toxic?" - choices = ["True", "False"] - target = ascii_uppercase[choices.index(record["gold"])] - return Sample(input=query, target=target, choices=choices) - - -civil_comments_LGBTQ = LightevalTaskConfig( - name="civil_comments:LGBTQ", - prompt_function=civil_comments_prompt, - hf_repo="lighteval/civil_comments_helm", - hf_subset="LGBTQ", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -civil_comments_black = LightevalTaskConfig( - name="civil_comments:black", - prompt_function=civil_comments_prompt, - hf_repo="lighteval/civil_comments_helm", - hf_subset="black", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -civil_comments_christian = LightevalTaskConfig( - name="civil_comments:christian", - prompt_function=civil_comments_prompt, - hf_repo="lighteval/civil_comments_helm", - hf_subset="christian", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -civil_comments_female = LightevalTaskConfig( - name="civil_comments:female", - prompt_function=civil_comments_prompt, - hf_repo="lighteval/civil_comments_helm", - hf_subset="female", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -civil_comments_male = LightevalTaskConfig( - name="civil_comments:male", - prompt_function=civil_comments_prompt, - hf_repo="lighteval/civil_comments_helm", - hf_subset="male", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -civil_comments_muslim = LightevalTaskConfig( - name="civil_comments:muslim", - prompt_function=civil_comments_prompt, - hf_repo="lighteval/civil_comments_helm", - hf_subset="muslim", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -civil_comments_other_religions = LightevalTaskConfig( - name="civil_comments:other_religions", - prompt_function=civil_comments_prompt, - hf_repo="lighteval/civil_comments_helm", - hf_subset="other_religions", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -civil_comments_white = LightevalTaskConfig( - name="civil_comments:white", - prompt_function=civil_comments_prompt, - hf_repo="lighteval/civil_comments_helm", - hf_subset="white", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -TASKS_TABLE = [ - civil_comments_LGBTQ, - civil_comments_black, - civil_comments_christian, - civil_comments_female, - civil_comments_male, - civil_comments_muslim, - civil_comments_other_religions, - civil_comments_white, -] diff --git a/src/lighteval/tasks/tasks/commonsenseqa.py b/src/lighteval/tasks/tasks/commonsenseqa.py deleted file mode 100644 index 59a1e6cdb..000000000 --- a/src/lighteval/tasks/tasks/commonsenseqa.py +++ /dev/null @@ -1,80 +0,0 @@ -""" -name: -Commonsenseqa - -dataset: -tau/commonsense_qa - -abstract: -CommonsenseQA is a new multiple-choice question answering dataset that requires -different types of commonsense knowledge to predict the correct answers . It -contains 12,102 questions with one correct answer and four distractor answers. -The dataset is provided in two major training/validation/testing set splits: -"Random split" which is the main evaluation split, and "Question token split", -see paper for details. - -languages: -english - -tags: -commonsense, multiple-choice, qa - -paper: -https://arxiv.org/abs/1811.00937 -""" - -from string import ascii_uppercase - -from inspect_ai.dataset import Sample -from inspect_ai.scorer import choice -from inspect_ai.solver import multiple_choice - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def commonsenseqa_prompt(line, task_name: str = None): - query = f"The following are multiple choice questions (with answers) about common sense.\nQuestion: {line['question']}\n" - query += "".join( - [f"{key}. {choice}\n" for key, choice in zip(ascii_uppercase, [f" {c}" for c in line["choices"]["text"]])] - ) - query += "Answer:" - - return Doc( - task_name=task_name, - query=query, - choices=list(ascii_uppercase)[: len(line["choices"]["text"])], - gold_index=list(ascii_uppercase).index(line["answerKey"].strip()), - instruction="The following are multiple choice questions (with answers) about common sense.\n", - ) - - -def record_to_sample(record): - query = record["question"] - choices = record["choices"]["text"] - target = record["answerKey"] - return Sample(input=query, target=target, choices=choices) - - -commonsenseqa = LightevalTaskConfig( - name="commonsenseqa", - prompt_function=commonsenseqa_prompt, - hf_repo="tau/commonsense_qa", - hf_subset="default", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -TASKS_TABLE = [ - commonsenseqa, -] diff --git a/src/lighteval/tasks/tasks/coqa.py b/src/lighteval/tasks/tasks/coqa.py deleted file mode 100644 index 34495ca3b..000000000 --- a/src/lighteval/tasks/tasks/coqa.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -name: -Coqa - -dataset: -stanfordnlp/coqa - -abstract: -CoQA is a large-scale dataset for building Conversational Question Answering -systems. The goal of the CoQA challenge is to measure the ability of machines to -understand a text passage and answer a series of interconnected questions that -appear in a conversation. - -languages: -english - -tags: -dialog, qa - -paper: -https://arxiv.org/abs/1808.07042 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def coqa_prompt(line, task_name: str = None): - results = [] - for q, a in zip(line["questions"], line["answers"]["input_text"]): - results.append(Doc(task_name=task_name, query=f"{line['story']} \n\nQ: {q}\n\nA: ", choices=[a], gold_index=0)) - return results - - -coqa_first_question = LightevalTaskConfig( - name="coqa", - prompt_function=coqa_prompt, - hf_repo="stanfordnlp/coqa", - hf_subset="default", - hf_avail_splits=["train", "validation"], - evaluation_splits=["validation"], - stop_sequence=["\n", "Question:", "question:"], - generation_size=100, - version=1, - metrics=[Metrics.exact_match], -) - -TASKS_TABLE = [ - coqa_first_question, -] diff --git a/src/lighteval/tasks/tasks/covid_dialogue.py b/src/lighteval/tasks/tasks/covid_dialogue.py deleted file mode 100644 index 4275dcaf8..000000000 --- a/src/lighteval/tasks/tasks/covid_dialogue.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -name: -Covid Dialogue - -dataset: -lighteval/covid_dialogue - -abstract: -The COVID-19 Dialogue dataset is a collection of 500+ dialogues between -doctors and patients during the COVID-19 pandemic. - -languages: -english - -tags: -dialog, medical - -paper: -https://arxiv.org/abs/2004.06561 -""" - -from inspect_ai.dataset import Sample -from inspect_ai.scorer import model_graded_fact -from inspect_ai.solver import generate, system_message - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -PROMPT = "Generate a response given a patient's questions and concerns." - - -def covid_dialogue_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"Generate a response given a patient's questions and concerns.\nPatient: {line['query']}\nDoctor: ", - choices=[line["answer"]], - gold_index=0, - instruction="Generate a response given a patient's questions and concerns.\n", - ) - - -def record_to_sample(record): - query = record["query"] - target = record["answer"] - return Sample(input=query, target=target) - - -covid_dialogue = LightevalTaskConfig( - name="covid_dialogue", - prompt_function=covid_dialogue_prompt, - hf_repo="lighteval/covid_dialogue", - hf_subset="default", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=128, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[system_message(PROMPT), generate(cache=True)], - scorer=model_graded_fact(), -) - -TASKS_TABLE = [ - covid_dialogue, -] diff --git a/src/lighteval/tasks/tasks/custom_task_classification_grammar_task.py b/src/lighteval/tasks/tasks/custom_task_classification_grammar_task.py deleted file mode 100644 index bcba9972f..000000000 --- a/src/lighteval/tasks/tasks/custom_task_classification_grammar_task.py +++ /dev/null @@ -1,418 +0,0 @@ -""" -name: -Emotion Classification - -dataset: -dair-ai/emotion - -abstract: -This task performs emotion classification classifying text into one of six -emotion categories: sadness, joy, love, anger, fear, surprise. - -languages: -english - -tags: -emotion, classification, multiple-choice - -paper: -""" - -import json -import logging -from typing import Any - -import numpy as np - -from lighteval.metrics.utils.metric_utils import SampleLevelMetricGrouping -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.lighteval_task import ( - LightevalTaskConfig, - TextGenerationInputGrammarType, -) -from lighteval.tasks.requests import Doc, SamplingMethod - - -logger = logging.getLogger(__name__) - -# Emotion labels for the emotion dataset from HuggingFace Hub -# These correspond to the 6-class emotion classification task with the following mapping: -# 0: sadness, 1: joy, 2: love, 3: anger, 4: fear, 5: surprise -EMOTION_LABELS = ["sadness", "joy", "love", "anger", "fear", "surprise"] - - -def parse_emotion_response(response: str | dict) -> dict[str, Any]: - """Parse the model's response into a standardized format. - - This function handles both JSON string and dictionary inputs, providing robust - parsing with validation against the predefined emotion labels. Invalid predictions - are automatically mapped to 'unknown' with appropriate logging. - - Args: - response (str | dict): The model's response, either as a JSON string - containing {"classification": "emotion_label"} or as a dictionary - with the same structure. - - Returns: - dict[str, Any]: Standardized dictionary containing: - - classification (str): The predicted emotion label, validated against - EMOTION_LABELS or 'unknown' if invalid/unparseable - - Examples: - >>> parse_emotion_response('{"classification": "joy"}') - {'classification': 'joy'} - - >>> parse_emotion_response({'classification': 'ANGER'}) - {'classification': 'anger'} - - >>> parse_emotion_response('{"classification": "invalid_emotion"}') - {'classification': 'unknown'} # with warning logged - - >>> parse_emotion_response('malformed json') - {'classification': 'unknown'} # with error logged - - Note: - - Case-insensitive matching: 'ANGER' and 'Anger' are normalized to 'anger' - - Whitespace is automatically stripped from predictions - - All parsing errors result in 'unknown' classification with detailed logging - """ - try: - # Handle dictionary input (already parsed JSON) - if isinstance(response, dict): - result = response - # Handle string input (JSON string that needs parsing) - else: - result = json.loads(response.strip()) - - # Extract and normalize the predicted emotion - predicted_emotion = result["classification"].lower().strip() - - # Validate that the prediction is one of the valid emotion labels - if predicted_emotion not in EMOTION_LABELS: - logger.warning( - f"Invalid emotion prediction: '{predicted_emotion}'. " - f"Expected one of {EMOTION_LABELS}. Using 'unknown'." - ) - predicted_emotion = "unknown" - - return { - "classification": predicted_emotion, - } - except (json.JSONDecodeError, KeyError, AttributeError, TypeError) as e: - # Handle specific parsing errors with detailed logging - logger.error( - f"Error parsing response: {str(e)}. Failed response was: {response}. Expected format: {{'classification': 'emotion_label'}}" - ) - return { - "classification": "unknown", - } - except Exception as e: - # Catch any other unexpected errors - logger.error(f"Unexpected error parsing response: {str(e)}. Failed response was: {response}") - return { - "classification": "unknown", - } - - -def emotion_classification_metric(model_response: ModelResponse, doc: Doc, **kwargs) -> dict[str, float]: - """Evaluate emotion classification predictions at the sample level. - - This function computes evaluation metrics for a single prediction, comparing - the model's emotion classification against the gold standard. It provides - detailed logging for debugging and tracks prediction quality. - - Args: - model_response (ModelResponse): The model's response containing generated text - in the text attribute, typically containing one prediction as either a - JSON string or dictionary with format {"classification": "emotion_label"} - doc (Doc): The document containing the query, choices, and gold - standard information. Must have gold_index attribute pointing to the - correct emotion label index. - **kwargs: Additional keyword arguments (unused but required for compatibility - with LightEval's metric interface) - - Returns: - dict[str, float]: Dictionary containing sample-level metrics: - - exact_match (float): 1.0 if prediction matches gold label, 0.0 otherwise - - unknown_prediction (float): 1.0 if prediction was 'unknown' (parsing - failure), 0.0 otherwise - - total_samples (float): Always 1.0 (count for this sample) - - Examples: - >>> doc = Doc(query="I'm so happy!", gold_index=2) # joy - >>> model_response = ModelResponse(text=['{"classification": "joy"}'], ...) - >>> result = emotion_classification_metric(model_response, doc) - >>> result - {'exact_match': 1.0, 'unknown_prediction': 0.0, 'total_samples': 1.0} - - >>> model_response = ModelResponse(text=['{"classification": "sadness"}'], ...) - >>> result = emotion_classification_metric(model_response, doc) - >>> result - {'exact_match': 0.0, 'unknown_prediction': 0.0, 'total_samples': 1.0} - - Note: - - The function expects exactly one prediction in the model_response.text list - - Gold labels are mapped from integer indices to emotion label strings - - All errors in prediction parsing result in 'unknown' classification - - Detailed logging is provided for debugging classification performance - """ - try: - # Parse the first (and typically only) prediction - prediction = parse_emotion_response(model_response.text[0]) - - # Map the gold label index to the corresponding emotion string - # The emotion dataset uses integer indices: 0=anger, 1=fear, 2=joy, etc. - gold_label_idx = doc.gold_index - expected_emotion = EMOTION_LABELS[gold_label_idx] - - # Log detailed information for debugging and analysis - logger.info("-" * 50) - logger.info("Processing new sample") - logger.info(f"- Text: {doc.query}") - logger.info(f"- Prediction: {prediction}") - logger.info(f"- Expected: {expected_emotion} (index: {gold_label_idx})") - - # Calculate evaluation metrics - is_exact_match = prediction["classification"] == expected_emotion - is_unknown = prediction["classification"] == "unknown" - - metrics = { - "exact_match": float(is_exact_match), - "unknown_prediction": float(is_unknown), - "total_samples": 1.0, - } - - logger.info(f"- Metrics: {metrics}") - if is_exact_match: - logger.info("✓ Correct prediction") - elif is_unknown: - logger.info("⚠ Parsing failure (unknown prediction)") - else: - logger.info("✗ Incorrect prediction") - logger.info("-" * 50) - - return metrics - - except (IndexError, KeyError) as e: - # Handle errors related to accessing gold label or prediction structure - logger.error(f"Error accessing gold label or prediction: {str(e)}") - logger.error(f"Gold index: {getattr(doc, 'gold_index', 'N/A')}") - logger.error(f"Raw prediction: {model_response.text[0] if model_response.text else 'Empty predictions'}") - return { - "exact_match": 0.0, - "unknown_prediction": 1.0, - "total_samples": 1.0, - } - except Exception as e: - # Handle any other unexpected errors - logger.error(f"Unexpected error processing prediction: {str(e)}") - logger.error(f"Raw prediction was: {model_response.text[0] if model_response.text else 'Empty predictions'}") - return { - "exact_match": 0.0, - "unknown_prediction": 1.0, - "total_samples": 1.0, - } - - -# Define the metric group for emotion classification evaluation -# This configures both sample-level and corpus-level metric calculations -emotion_classification_group = SampleLevelMetricGrouping( - metric_name=[ - "exact_match", # Primary accuracy metric - "unknown_prediction", # Tracks parsing failures - "total_samples", # Sample count for aggregation - ], - higher_is_better={ - "exact_match": True, # Higher accuracy is better - "unknown_prediction": False, # Fewer parsing failures is better - "total_samples": True, # More samples processed is better - }, - category=SamplingMethod.GENERATIVE, # Classification via text generation - sample_level_fn=emotion_classification_metric, # Function for individual samples - corpus_level_fn={ - "exact_match": np.mean, # Average accuracy across all samples - "unknown_prediction": np.mean, # Proportion of parsing failures - "total_samples": np.sum, # Total number of samples processed - }, -) - - -def prompt_emotion_classification(line: dict[str, Any], task_name: str = None) -> Doc: - """Format the emotion classification task with detailed prompt engineering. - - This function converts a single sample from the emotion dataset into a structured - prompt that provides clear instructions and emotion definitions to improve - classification accuracy. The prompt includes detailed explanations of each - emotion category to reduce ambiguity. - - Args: - line (dict[str, Any]): A single sample from the emotion dataset containing: - - 'text' (str): The input text to classify - - 'label' (int): The gold standard emotion label (0-5) - task_name (str, optional): Name of the task for identification purposes. - Defaults to None. - - Returns: - Doc: A formatted document object containing: - - task_name: Task identifier - - query: The formatted prompt with text and emotion definitions - - choices: List of available emotion labels - - gold_index: The correct emotion label index - - instruction: Empty string (instructions are embedded in query) - - Examples: - >>> line = {'text': 'I am so excited for tomorrow!', 'label': 2} - >>> doc = prompt_emotion_classification(line, 'emotion_test') - >>> print(doc.query) - Classify the emotion expressed in the following text: "I am so excited for tomorrow!" - ... - >>> doc.gold_index - 2 - >>> doc.choices - ['sadness', 'joy', 'love', 'anger', 'fear', 'surprise'] - - Note: - - The prompt includes detailed definitions for each emotion to improve accuracy - - Emotion definitions are based on common psychological categorizations - - The format is optimized for both human readability and model understanding - """ - # Extract the text to be classified - text = line["text"] - - # Create a comprehensive classification prompt with detailed emotion definitions - # This approach helps models understand the subtle differences between emotions - prompt = f"""Classify the emotion expressed in the following text: "{text}" - -Available emotion labels and their meanings: -- sadness: Feeling of sorrow, grief, or unhappiness. Covers melancholy, disappointment, - loss, or general negative emotional states related to unfortunate circumstances. -- joy: Feeling of happiness, delight, or pleasure. Encompasses positive emotions like - excitement, satisfaction, contentment, and general well-being. -- love: Feeling of affection, care, or romantic attachment. Includes expressions of - deep fondness, romantic interest, or strong positive feelings toward people or things. -- anger: Feeling of displeasure, hostility, or annoyance. Often involves frustration, - irritation, or aggressive sentiments toward people, situations, or objects. -- fear: Feeling of anxiety, worry, or being afraid. Includes nervousness, concern - about future events, or apprehension about potential threats or negative outcomes. -- surprise: Feeling of astonishment or being caught off guard. Includes unexpected - reactions, amazement, or responses to sudden or unanticipated events. - -Choose the emotion that best matches the sentiment expressed in the text.""" - - return Doc( - task_name=task_name, - query=prompt, - choices=EMOTION_LABELS, # Available emotion label options - gold_index=line["label"], # Gold standard emotion index (0-5) - instruction="", # Instructions are embedded in the query - ) - - -def get_emotion_classification_grammar() -> TextGenerationInputGrammarType: - """Define the JSON schema grammar for constrained emotion classification responses. - - This function creates a strict JSON schema that constrains the model's output - to only valid emotion labels, preventing hallucination and ensuring consistent - response format. The grammar constraint is enforced during text generation. - - Returns: - TextGenerationInputGrammarType: A JSON schema grammar specification that: - - Enforces JSON object structure with required "classification" field - - Constrains classification values to only valid emotion labels - - Ensures consistent response parsing across different models - - Schema Structure: - { - "type": "object", - "properties": { - "classification": { - "type": "string", - "description": "Emotion classification", - "enum": ["anger", "fear", "joy", "love", "sadness", "surprise"] - } - }, - "required": ["classification"] - } - - Examples: - Valid responses that match this grammar: - - {"classification": "joy"} - - {"classification": "anger"} - - Invalid responses that would be rejected: - - {"emotion": "joy"} # Wrong field name - - {"classification": "happy"} # Invalid emotion label - - "joy" # Not a JSON object - - Note: - - This grammar constraint significantly improves response consistency - - It prevents the model from generating invalid emotion labels - - Compatible with grammar-enabled backends like vLLM, TGI, and others - - The enum constraint is crucial for maintaining label consistency - """ - return TextGenerationInputGrammarType( - type="json", # Specify JSON schema grammar type - value={ - "type": "object", # Require JSON object structure - "properties": { - "classification": { - "type": "string", # Classification must be a string - "description": "Emotion classification from the provided list", - "enum": EMOTION_LABELS, # Strictly constrain to valid emotion labels only - }, - }, - "required": ["classification"], # Classification field is mandatory - "additionalProperties": False, # Prevent extra fields in response - }, - ) - - -# Task configuration for emotion classification using the HuggingFace emotion dataset -# This configuration optimizes for accuracy while maintaining efficient resource usage -EMOTION_CLASSIFICATION_TASK = LightevalTaskConfig( - name="emotion_classification", # Unique task identifier - prompt_function=prompt_emotion_classification, # Custom prompt formatting function - # Classification as a community/custom task - hf_repo="emotion", # HuggingFace Hub dataset repository - hf_subset=None, # Use default subset (no subset specified) - metrics=[emotion_classification_group], # Evaluation metrics configuration - generation_size=64, # Conservative token limit for JSON responses (~30-40 tokens typical) - generation_grammar=get_emotion_classification_grammar(), # JSON schema constraint - stop_sequence=["\n\n"], # Early stopping on double newline - evaluation_splits=["test"], # Evaluate on test split only - hf_avail_splits=["train", "validation", "test"], # Available dataset splits -) - -# Export the task for LightEval discovery -# This list is automatically detected by LightEval when loading custom tasks -TASKS_TABLE = [EMOTION_CLASSIFICATION_TASK] - -# Development and testing utilities -if __name__ == "__main__": - # Print available tasks for verification - print("Available tasks:", [t.name for t in TASKS_TABLE]) - print("Total tasks:", len(TASKS_TABLE)) - - # Print task configuration summary for debugging - task = TASKS_TABLE[0] - print("\nTask Configuration Summary:") - print(f" Name: {task.name}") - print(f" Dataset: {task.hf_repo}") - print(f" Splits: {task.evaluation_splits}") - print(f" Metrics: {[m.metric_name for m in task.metric]}") - print(f" Generation size: {task.generation_size}") - print(f" Grammar constrained: {task.generation_grammar is not None}") - print(f" Stop sequences: {task.stop_sequence}") - - # Verify emotion labels configuration - print(f"\nEmotion Labels ({len(EMOTION_LABELS)}):") - for i, label in enumerate(EMOTION_LABELS): - print(f" {i}: {label}") - - print("\nUsage Examples:") - print( - f" TGI: uv run lighteval endpoint tgi config/tgi/tgi.yaml 'custom|{task.name}|0' --custom-tasks {__file__} --output-dir results --override-batch-size 1 --use-chat-template --save-details --no-public-run --max-samples 10" - ) - print( - f" Full: uv run lighteval endpoint tgi config/tgi/tgi.yaml 'custom|{task.name}|5' --custom-tasks {__file__} --output-dir results --override-batch-size 1 --use-chat-template --save-details --no-public-run" - ) diff --git a/src/lighteval/tasks/tasks/drop_qa.py b/src/lighteval/tasks/tasks/drop_qa.py deleted file mode 100644 index 91a6adff7..000000000 --- a/src/lighteval/tasks/tasks/drop_qa.py +++ /dev/null @@ -1,84 +0,0 @@ -""" -name: -Drop Qa - -dataset: -lighteval/drop_harness - -abstract: -The DROP dataset is a new question-answering dataset designed to evaluate the -ability of language models to answer complex questions that require reasoning -over multiple sentences. - -languages: -english - -tags: -math, qa, reasoning - -paper: -https://arxiv.org/abs/1810.00505 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def drop_prompt(line, task_name: str = None): - def _flatten_validated_answers(validated_answers): - valid_answers = [] - for i in range(len(validated_answers["number"])): - valid_answers.append( - { - "number": validated_answers["number"][i], - "date": validated_answers["date"][i], - "spans": validated_answers["spans"][i], - } - ) - return valid_answers - - def parse_answer(answer): - if answer["number"] != "": - return (str(answer["number"]),) - if answer["spans"] != []: - return tuple(answer["spans"]) - return (" ".join([answer["date"]["day"], answer["date"]["month"], answer["date"]["year"]]).strip(),) - - answers = [] - answers_set = set() - candidates = [line["answer"]] + _flatten_validated_answers(line["validated_answers"]) - for candidate in candidates: - answer = parse_answer(candidate) - if answer in answers_set: - continue - answers.append(answer) - answers_set.add(answer) - - is_few_shots = line.get("__few_shots", False) - - return Doc( - task_name=task_name, - query=f"Passage: {line['passage']}\nQuestion: {line['question']}\nAnswer:", - choices=[f"{' ' if is_few_shots else ''}{', '.join(a)}" for a in answers], - gold_index=list(range(len(answers))), - specific={"golds_no_preprocessing": [list(a) for a in answers]}, - ) - - -drop_qa = LightevalTaskConfig( - name="drop", - prompt_function=drop_prompt, - hf_repo="lighteval/drop_harness", - hf_subset="default", - evaluation_splits=("validation",), - few_shots_split="train", - generation_size=250, - stop_sequence=["Question:", "question:", "\n"], - metrics=[Metrics.exact_match], - version=1, -) - -TASKS_TABLE = [ - drop_qa, -] diff --git a/src/lighteval/tasks/tasks/dyck_language.py b/src/lighteval/tasks/tasks/dyck_language.py deleted file mode 100644 index 6342a9f9c..000000000 --- a/src/lighteval/tasks/tasks/dyck_language.py +++ /dev/null @@ -1,107 +0,0 @@ -""" -name: -Dyck Language - -dataset: -lighteval/DyckLanguage - -abstract: -Scenario testing hierarchical reasoning through the Dyck formal languages. - -languages: -english - -tags: -reasoning - -paper: -https://aclanthology.org/W19-3905/ -""" - -from inspect_ai.dataset import Sample -from inspect_ai.scorer import exact -from inspect_ai.solver import generate - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -PROMPT = "Please complete the rest of the following Dyck sequences, making sure that the parentheses are closed properly.\n Input: {prompt}" - - -def record_to_sample(record): - return Sample(input=PROMPT.format(prompt=record["input"]), target=record["output"]) - - -def dyck_language_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"Please complete the rest of the following Dyck sequences, making sure that the parentheses are closed properly.\n Input: {line['input']}", - choices=[line["output"]], - gold_index=0, - instruction="Please complete the rest of the following Dyck sequences, making sure that the parentheses are closed properly.\n ", - ) - - -dyck_language_2 = LightevalTaskConfig( - name="dyck_language:2", - prompt_function=dyck_language_prompt, - hf_repo="lighteval/DyckLanguage", - hf_subset="2", - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=exact(), - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - - -dyck_language_3 = LightevalTaskConfig( - name="dyck_language:3", - prompt_function=dyck_language_prompt, - hf_repo="lighteval/DyckLanguage", - hf_subset="3", - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=exact(), - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - - -dyck_language_4 = LightevalTaskConfig( - name="dyck_language:4", - prompt_function=dyck_language_prompt, - hf_repo="lighteval/DyckLanguage", - hf_subset="4", - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=exact(), - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - dyck_language_2, - dyck_language_3, - dyck_language_4, -] diff --git a/src/lighteval/tasks/tasks/entity_data_imputation.py b/src/lighteval/tasks/tasks/entity_data_imputation.py deleted file mode 100644 index 0999c83b7..000000000 --- a/src/lighteval/tasks/tasks/entity_data_imputation.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -name: -Entity Data Imputation - -dataset: -lighteval/Buy, lighteval/Restaurant - -abstract: -Scenario that tests the ability to impute missing entities in a data table. - -languages: -english - -tags: -reasoning - -paper: -https://ieeexplore.ieee.org/document/9458712 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def entity_data_imputation_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"What is the missing value?\n{line['text']}\nAnswer:", - choices=[line["gold"]], - gold_index=0, - instruction="What is the missing value?\n", - ) - - -entity_data_imputation_Buy = LightevalTaskConfig( - name="entity_data_imputation:Buy", - prompt_function=entity_data_imputation_prompt, - hf_repo="lighteval/Buy", - hf_subset="default", - hf_avail_splits=["train", "test", "valid"], - evaluation_splits=["valid", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - - -entity_data_imputation_Restaurant = LightevalTaskConfig( - name="entity_data_imputation:Restaurant", - prompt_function=entity_data_imputation_prompt, - hf_repo="lighteval/Restaurant", - hf_subset="default", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - entity_data_imputation_Buy, - entity_data_imputation_Restaurant, -] diff --git a/src/lighteval/tasks/tasks/entitymatching.py b/src/lighteval/tasks/tasks/entitymatching.py deleted file mode 100644 index 252559c8d..000000000 --- a/src/lighteval/tasks/tasks/entitymatching.py +++ /dev/null @@ -1,245 +0,0 @@ -""" -name: -Entitymatching - -dataset: -lighteval/EntityMatching - -abstract: -Simple entity matching benchmark. - -languages: -english - -tags: -classification, reasoning - -paper: -https://dl.acm.org/doi/10.14778/3007263.3007314 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def entity_matching_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"Are Product A and Product B the same? Yes or No?\nProduct A is {line['productA']}. Product B is {line['productB']}. Are A and B the same?\nAnswer:", - choices=["No", "Yes"], - gold_index=int(line["same"]), - instruction="Are Product A and Product B the same? Yes or No?\n", - ) - - -entity_matching_Abt_Buy = LightevalTaskConfig( - name="entity_matching:Abt_Buy", - prompt_function=entity_matching_prompt, - hf_repo="lighteval/EntityMatching", - hf_subset="Abt_Buy", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -entity_matching_Amazon_Google = LightevalTaskConfig( - name="entity_matching:Amazon_Google", - prompt_function=entity_matching_prompt, - hf_repo="lighteval/EntityMatching", - hf_subset="Amazon_Google", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -entity_matching_Beer = LightevalTaskConfig( - name="entity_matching:Beer", - prompt_function=entity_matching_prompt, - hf_repo="lighteval/EntityMatching", - hf_subset="Beer", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -entity_matching_Company = LightevalTaskConfig( - name="entity_matching:Company", - prompt_function=entity_matching_prompt, - hf_repo="lighteval/EntityMatching", - hf_subset="Company", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -entity_matching_DBLP_ACM = LightevalTaskConfig( - name="entity_matching:DBLP_ACM", - prompt_function=entity_matching_prompt, - hf_repo="lighteval/EntityMatching", - hf_subset="DBLP_ACM", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -entity_matching_DBLP_GoogleScholar = LightevalTaskConfig( - name="entity_matching:DBLP_GoogleScholar", - prompt_function=entity_matching_prompt, - hf_repo="lighteval/EntityMatching", - hf_subset="DBLP_GoogleScholar", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -entity_matching_Dirty_DBLP_ACM = LightevalTaskConfig( - name="entity_matching:Dirty_DBLP_ACM", - prompt_function=entity_matching_prompt, - hf_repo="lighteval/EntityMatching", - hf_subset="Dirty_DBLP_ACM", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -entity_matching_Dirty_DBLP_GoogleScholar = LightevalTaskConfig( - name="entity_matching:Dirty_DBLP_GoogleScholar", - prompt_function=entity_matching_prompt, - hf_repo="lighteval/EntityMatching", - hf_subset="Dirty_DBLP_GoogleScholar", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -entity_matching_Dirty_Walmart_Amazon = LightevalTaskConfig( - name="entity_matching:Dirty_Walmart_Amazon", - prompt_function=entity_matching_prompt, - hf_repo="lighteval/EntityMatching", - hf_subset="Dirty_Walmart_Amazon", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -entity_matching_Dirty_iTunes_Amazon = LightevalTaskConfig( - name="entity_matching:Dirty_iTunes_Amazon", - prompt_function=entity_matching_prompt, - hf_repo="lighteval/EntityMatching", - hf_subset="Dirty_iTunes_Amazon", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -entity_matching_Fodors_Zagats = LightevalTaskConfig( - name="entity_matching=Fodors_Zagats", - prompt_function=entity_matching_prompt, - hf_repo="lighteval/EntityMatching", - hf_subset="Fodors_Zagats", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -entity_matching_Walmart_Amazon = LightevalTaskConfig( - name="entity_matching:Walmart_Amazon", - prompt_function=entity_matching_prompt, - hf_repo="lighteval/EntityMatching", - hf_subset="Walmart_Amazon", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -entity_matching_iTunes_Amazon = LightevalTaskConfig( - name="entity_matching:iTunes_Amazon", - prompt_function=entity_matching_prompt, - hf_repo="lighteval/EntityMatching", - hf_subset="iTunes_Amazon", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - entity_matching_Abt_Buy, - entity_matching_Amazon_Google, - entity_matching_Beer, - entity_matching_Company, - entity_matching_DBLP_ACM, - entity_matching_DBLP_GoogleScholar, - entity_matching_Dirty_DBLP_ACM, - entity_matching_Dirty_DBLP_GoogleScholar, - entity_matching_Dirty_Walmart_Amazon, - entity_matching_Dirty_iTunes_Amazon, - entity_matching_Fodors_Zagats, - entity_matching_Walmart_Amazon, - entity_matching_iTunes_Amazon, -] diff --git a/src/lighteval/tasks/tasks/ethics.py b/src/lighteval/tasks/tasks/ethics.py deleted file mode 100644 index 07471fb5f..000000000 --- a/src/lighteval/tasks/tasks/ethics.py +++ /dev/null @@ -1,149 +0,0 @@ -""" -name: -Ethics - -dataset: -lighteval/hendrycks_ethics - -abstract: -The Ethics benchmark for evaluating the ability of language models to reason about -ethical issues. - -languages: -english - -tags: -classification, ethics, justice, morality, utilitarianism, virtue - -paper: -https://arxiv.org/abs/2008.02275 -""" - -import random - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def ethics_commonsense_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"{line['input']}\nQuestion: Is this wrong?\nAnswer:", - choices=[" no", " yes"], - gold_index=int(line["label"]), - ) - - -def ethics_deontology_prompt(line, task_name: str = None): - return Doc(task_name=task_name, query=line["input"], choices=[line["label"]], gold_index=0) - - -def ethics_justice_prompt(line, task_name: str = None): - return Doc(task_name=task_name, query=line["input"], choices=[line["label"]], gold_index=0) - - -def ethics_utilitarianism_prompt(line, task_name: str = None): - rnd = random.Random(line["activity"]) - scenarios = [line["activity"], line["baseline"]] - ordering = [0, 1] - rnd.shuffle(ordering) - return Doc( - task_name=task_name, - query=f"Scenario 1: {scenarios[ordering[0]]}\nScenario 2: {scenarios[ordering[1]]}\nQuestion: Is Scenario 1 preferable?\nAnswer:", - choices=[" no", " yes"], - gold_index=int(ordering.index(0) == 0), - ) - - -def ethics_virtue_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f'Sentence: {line["scenario"]}\nQuestion: Does the character in this sentence exhibit the trait "{line["trait"]}"?\nAnswer:', - choices=[" no", " yes"], - gold_index=int(line["label"]), - ) - - -ethics_commonsense = LightevalTaskConfig( - name="ethics:commonsense", - prompt_function=ethics_commonsense_prompt, - hf_repo="lighteval/hendrycks_ethics", - hf_subset="commonsense", - hf_avail_splits=["train", "validation", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -ethics_deontology = LightevalTaskConfig( - name="ethics:deontology", - prompt_function=ethics_deontology_prompt, - hf_repo="lighteval/hendrycks_ethics", - hf_subset="deontology", - hf_avail_splits=["train", "validation", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -ethics_justice = LightevalTaskConfig( - name="ethics:justice", - prompt_function=ethics_justice_prompt, - hf_repo="lighteval/hendrycks_ethics", - hf_subset="justice", - hf_avail_splits=["train", "validation", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -ethics_utilitarianism = LightevalTaskConfig( - name="ethics:utilitarianism", - prompt_function=ethics_utilitarianism_prompt, - hf_repo="lighteval/hendrycks_ethics", - hf_subset="utilitarianism", - hf_avail_splits=["train", "validation", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -ethics_virtue = LightevalTaskConfig( - name="ethics:virtue", - prompt_function=ethics_virtue_prompt, - hf_repo="lighteval/hendrycks_ethics", - hf_subset="virtue", - hf_avail_splits=["train", "validation", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - ethics_commonsense, - ethics_deontology, - ethics_justice, - ethics_utilitarianism, - ethics_virtue, -] diff --git a/src/lighteval/tasks/tasks/glue.py b/src/lighteval/tasks/tasks/glue.py deleted file mode 100644 index 984ce92c3..000000000 --- a/src/lighteval/tasks/tasks/glue.py +++ /dev/null @@ -1,448 +0,0 @@ -""" -name: -GLUE - -dataset: -nyu-mll/glue, aps/super_glue - -abstract: -The General Language Understanding Evaluation (GLUE) benchmark is a collection -of resources for training, evaluating, and analyzing natural language -understanding systems. - -languages: -english - -tags: -classification, language-understanding - -paper: -""" - -import re - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def boolq_harness_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"{line['passage']}\nQuestion: {line['question']}?\nAnswer:", - choices=[" no", " yes"], - gold_index=int(line["label"]), - ) - - -def cb_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"{line['premise']}\nQuestion: {line['hypothesis']}. True, False or Neither?\nAnswer:", - choices=[" True", " False", " Neither"], - gold_index=int(line["label"]), - ) - - -def copa_prompt(line, task_name: str = None): - connector = {"cause": "because", "effect": "therefore"}[line["question"]] - return Doc( - task_name=task_name, - query=f"{line['premise'].strip()[:-1]} {connector}", - choices=[f" {line[c][0].lower()}{line[c][1:]}" for c in ["choice1", "choice2"]], - gold_index=int(line["label"]), - ) - - -def multirc_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"{line['paragraph']}\nQuestion: {line['question']}\nAnswer:", - choices=[f" {line['answer']}\nIs the answer correct? yes", f" {line['answer']}\nIs the answer correct? no"], - gold_index=0 if line["label"] else 1, - ) - - -def wic_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"Sentence 1: {line['sentence1']}\nSentence 2: {line['sentence2']}\nQuestion: Is the word '{line['word']}' used in the same way in the two sentences above?\nAnswer:", - choices=[" no", " yes"], - gold_index=int(line["label"]), - ) - - -def wsc_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"Passage: {line['text']}\n'Question: In the passage above, does the pronoun {line['span2_text']} refer to {line['span1_text']}?\nAnswer:", - choices=[" no", " yes"], - gold_index=int(line["label"]), - ) - - -def cola_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"{line['sentence']}\nQuestion: Does this sentence make sense?\nAnswer:", - choices=[" no", " yes"], - gold_index=int(line["label"]), - ) - - -def mnli_prompt(line, task_name: str = None): - hypothesis = line["hypothesis"].strip() + ("" if line["hypothesis"].strip().endswith(".") else ".") - return Doc( - task_name=task_name, - query=f"{line['premise']}\nQuestion: {hypothesis} True, False or Neither?\nAnswer:", - choices=[" True", " Neither", " False"], - gold_index=int(line["label"]), - ) - - -def mrpc_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"Sentence 1: {line['sentence1']}\nSentence 2: {line['sentence2']}\nQuestion: Do both sentences mean the same thing?\nAnswer:", - choices=[" no", " yes"], - gold_index=int(line["label"]), - ) - - -def qnli_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"{line['question']}\n{line['sentence']}\nQuestion: Does this response answer the question?\nAnswer:", - choices=[" yes", " no"], - gold_index=int(line["label"]), - ) - - -def qqp_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"Question 1: {line['question1']}\nQuestion 2: {line['question2']}\nQuestion: Do both questions ask the same thing?\nAnswer:", - choices=[" no", " yes"], - gold_index=int(line["label"]), - ) - - -def rte_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"{line['sentence1']}\nQuestion: {line['sentence2']} True or False?\nAnswer:", - choices=[" True", " False"], - gold_index=int(line["label"]), - ) - - -def sst_prompt(line, task_name: str = None): - def general_detokenize(cur_string): - cur_string = cur_string.replace(" n't", "n't") - cur_string = cur_string.replace(" )", ")") - cur_string = cur_string.replace("( ", "(") - cur_string = cur_string.replace('" ', '"') - cur_string = cur_string.replace(' "', '"') - cur_string = re.sub(r" (['.,])", r"\1", cur_string) - return cur_string - - return Doc( - task_name=task_name, - query=f"{general_detokenize(line['sentence'])}\nQuestion: Is this sentence positive or negative?\nAnswer:", - choices=[" negative", " positive"], - gold_index=int(line["label"]), - ) - - -def stsb_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"sentence 1: {line['sentence1']}\nsentence 2: {line['sentence2']}\nOn a scale of 0 to 5, how similar are the two sentences?\nAnswer:", - gold_index=0, - choices=[line["label"]], - ) - - -def wnli_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"{line['sentence1']}\nQuestion: {line['sentence2']} True or False?\nAnswer:", - choices=[" False", " True"], - gold_index=int(line["label"]), - ) - - -glue_cola = LightevalTaskConfig( - name="glue:cola", - prompt_function=cola_prompt, - hf_repo="nyu-mll/glue", - hf_subset="cola", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc, Metrics.mcc], - stop_sequence=["\n"], - version=0, -) - -glue_mnli = LightevalTaskConfig( - name="glue:mnli", - prompt_function=mnli_prompt, - hf_repo="nyu-mll/glue", - hf_subset="mnli_matched", - hf_avail_splits=["train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -glue_mnli_mismatched = LightevalTaskConfig( - name="glue:mnli_mismatched", - prompt_function=mnli_prompt, - hf_repo="nyu-mll/glue", - hf_subset="mnli_mismatched", - hf_avail_splits=["train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -glue_mrpc = LightevalTaskConfig( - name="glue:mrpc", - prompt_function=mrpc_prompt, - hf_repo="nyu-mll/glue", - hf_subset="mrpc", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc, Metrics.loglikelihood_f1], - stop_sequence=["\n"], - version=0, -) - -glue_qnli = LightevalTaskConfig( - name="glue:qnli", - prompt_function=qnli_prompt, - hf_repo="nyu-mll/glue", - hf_subset="qnli", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -glue_qqp = LightevalTaskConfig( - name="glue:qqp", - prompt_function=qqp_prompt, - hf_repo="nyu-mll/glue", - hf_subset="qqp", - hf_avail_splits=["train", "validation", "test"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc, Metrics.loglikelihood_f1], - stop_sequence=["\n"], - version=0, -) - -glue_rte = LightevalTaskConfig( - name="glue:rte", - prompt_function=rte_prompt, - hf_repo="nyu-mll/glue", - hf_subset="rte", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -glue_sst2 = LightevalTaskConfig( - name="glue:sst2", - prompt_function=sst_prompt, - hf_repo="nyu-mll/glue", - hf_subset="sst2", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -glue_stsb = LightevalTaskConfig( - name="glue:stsb", - prompt_function=stsb_prompt, - hf_repo="nyu-mll/glue", - hf_subset="stsb", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -glue_wnli = LightevalTaskConfig( - name="glue:wnli", - prompt_function=wnli_prompt, - hf_repo="nyu-mll/glue", - hf_subset="wnli", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -super_glue_boolq = LightevalTaskConfig( - name="super_glue:boolq", - prompt_function=boolq_harness_prompt, - hf_repo="aps/super_glue", - hf_subset="boolq", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -super_glue_cb = LightevalTaskConfig( - name="super_glue:cb", - prompt_function=cb_prompt, - hf_repo="aps/super_glue", - hf_subset="cb", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc, Metrics.multi_f1_numeric], - stop_sequence=["\n"], - version=0, -) - -super_glue_copa = LightevalTaskConfig( - name="super_glue:copa", - prompt_function=copa_prompt, - hf_repo="aps/super_glue", - hf_subset="copa", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -super_glue_rte = LightevalTaskConfig( - name="super_glue:rte", - prompt_function=rte_prompt, - hf_repo="aps/super_glue", - hf_subset="rte", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -super_glue_multirc = LightevalTaskConfig( - name="super_glue:multirc", - prompt_function=multirc_prompt, - hf_repo="aps/super_glue", - hf_subset="multirc", - hf_avail_splits=["train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -super_glue_wic = LightevalTaskConfig( - name="super_glue:wic", - prompt_function=wic_prompt, - hf_repo="aps/super_glue", - hf_subset="wic", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -super_glue_wsc = LightevalTaskConfig( - name="super_glue:wsc", - prompt_function=wsc_prompt, - hf_repo="aps/super_glue", - hf_subset="wsc", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - glue_cola, - glue_mnli, - glue_mnli_mismatched, - glue_mrpc, - glue_qnli, - glue_qqp, - glue_rte, - glue_sst2, - glue_stsb, - glue_wnli, - super_glue_boolq, - super_glue_cb, - super_glue_copa, - super_glue_rte, - super_glue_multirc, - super_glue_wic, - super_glue_wsc, -] diff --git a/src/lighteval/tasks/tasks/gpqa.py b/src/lighteval/tasks/tasks/gpqa.py deleted file mode 100644 index 354d6addf..000000000 --- a/src/lighteval/tasks/tasks/gpqa.py +++ /dev/null @@ -1,189 +0,0 @@ -""" -name: -Gpqa - -dataset: -Idavidrein/gpqa - -abstract: -GPQA is a dataset of 448 expert-written multiple-choice questions in biology, -physics, and chemistry, designed to test graduate-level reasoning. The questions -are extremely difficult—PhD-level experts score about 65%, skilled non-experts -34% (even with web access), and GPT-4 around 39%. GPQA aims to support research -on scalable oversight, helping humans evaluate and trust AI systems that may -exceed human expertise. - -languages: -english - -tags: -biology, chemistry, graduate-level, multiple-choice, physics, qa, reasoning, science - -paper: -https://arxiv.org/abs/2311.12022 - -starred: -true -""" - -import random -from string import ascii_uppercase - -from inspect_ai.dataset import Sample -from inspect_ai.scorer import choice -from inspect_ai.solver import multiple_choice - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def record_to_sample(record): - gold_index = random.randint(0, 3) - choices = [record["Incorrect Answer 1"], record["Incorrect Answer 2"], record["Incorrect Answer 3"]] - choices.insert(gold_index, record["Correct Answer"]) - return Sample( - input=record["Question"].strip(), - choices=choices, - target=ascii_uppercase[gold_index], - ) - - -def sample_to_fewshot(sample): - return f"{sample.input}\n\n" + f"ANSWER: {sample.target}" - - -def gpqa_prompt(line, task_name: str = None): - GPQA_QUERY_TEMPLATE = """ -Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering. - -{Question} - -A) {A} -B) {B} -C) {C} -D) {D} -""".strip() - gold_index = random.randint(0, 3) - choices = [line["Incorrect Answer 1"], line["Incorrect Answer 2"], line["Incorrect Answer 3"]] - choices.insert(gold_index, line["Correct Answer"]) - - query = GPQA_QUERY_TEMPLATE.format( - A=choices[0], B=choices[1], C=choices[2], D=choices[3], Question=line["Question"] - ) - - return Doc( - task_name=task_name, - query=query, - choices=list(ascii_uppercase)[: len(choices)], - gold_index=gold_index, - instruction=query, - ) - - -def gpqa_instruct_prompt(line, task_name: str = None): - gold_index = random.randint(0, 3) - choices = [line["Incorrect Answer 1"], line["Incorrect Answer 2"], line["Incorrect Answer 3"]] - choices.insert(gold_index, line["Correct Answer"]) - instruction = "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering." - query_template = "{Instruction}\n\n{Question}\n\nA) {A}\nB) {B}\nC) {C}\nD) {D}" - query = query_template.format( - A=choices[0].strip(), - B=choices[1].strip(), - C=choices[2].strip(), - D=choices[3].strip(), - Question=line["Question"].strip(), - Instruction=instruction, - ) - - return Doc( - task_name=task_name, - query=query, - choices=list(ascii_uppercase)[: len(choices)], - gold_index=gold_index, - instruction=instruction, - ) - - -gpqa = LightevalTaskConfig( - name="gpqa:mc", - prompt_function=gpqa_prompt, - sample_fields=record_to_sample, - sample_to_fewshot=sample_to_fewshot, - solver=[multiple_choice(cache=True)], - scorer=choice(), - hf_repo="Idavidrein/gpqa", - hf_subset="gpqa_main", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -gpqa_diamond_instruct = LightevalTaskConfig( - name="gpqa:diamond", - prompt_function=gpqa_instruct_prompt, - sample_fields=record_to_sample, - sample_to_fewshot=sample_to_fewshot, - solver=[multiple_choice(cache=True)], - scorer=choice(), - hf_repo="Idavidrein/gpqa", - hf_subset="gpqa_diamond", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=32768, # needed for reasoning models like R1 - metrics=[Metrics.gpqa_instruct_pass_at_k(sample_params={"k": 1})], - stop_sequence=[], # no stop sequence, will use eos token - version=1, -) - -gpqa_extended_instruct = LightevalTaskConfig( - name="gpqa:extended", - prompt_function=gpqa_instruct_prompt, - sample_fields=record_to_sample, - sample_to_fewshot=sample_to_fewshot, - solver=[multiple_choice(cache=True)], - scorer=choice(), - hf_repo="Idavidrein/gpqa", - hf_subset="gpqa_extended", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=32768, # needed for reasoning models like R1 - metrics=[Metrics.gpqa_instruct_metric], - stop_sequence=[], # no stop sequence, will use eos token - version=0, -) - -gpqa_main_instruct = LightevalTaskConfig( - name="gpqa:main", - prompt_function=gpqa_instruct_prompt, - sample_fields=record_to_sample, - sample_to_fewshot=sample_to_fewshot, - solver=[multiple_choice(cache=True)], - scorer=choice(), - hf_repo="Idavidrein/gpqa", - hf_subset="gpqa_main", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=32768, # needed for reasoning models like R1 - metrics=[Metrics.gpqa_instruct_metric], - stop_sequence=[], # no stop sequence, will use eos token - version=0, -) - -TASKS_TABLE = [ - gpqa, - gpqa_diamond_instruct, - gpqa_extended_instruct, - gpqa_main_instruct, -] diff --git a/src/lighteval/tasks/tasks/gsm8k.py b/src/lighteval/tasks/tasks/gsm8k.py deleted file mode 100644 index 24c98055e..000000000 --- a/src/lighteval/tasks/tasks/gsm8k.py +++ /dev/null @@ -1,90 +0,0 @@ -""" -name: -Gsm8K - -dataset: -openai/gsm8k - -abstract: -GSM8K is a dataset of 8,000+ high-quality, single-step arithmetic word problems. - -languages: -english - -tags: -math, reasoning - -paper: -https://arxiv.org/abs/2110.14168 -""" - -from inspect_ai.dataset import Sample -from inspect_ai.solver import generate, prompt_template - -from lighteval.metrics.metrics import Metrics, math_scorer -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -# setup for problem + instructions for providing answer -MATH_PROMPT_TEMPLATE = """ -Solve the following math problem step by step. The last line of your -response should be of the form "ANSWER: $ANSWER" (without quotes) -where $ANSWER is the answer to the problem. - -{prompt} - -Remember to put your answer on its own line at the end in the form -"ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to -the problem, and you do not need to use a \\boxed command. - -Reasoning: -""".strip() - - -def record_to_sample(record): - DELIM = "####" - input = record["question"] - answer = record["answer"].split(DELIM) - target = answer.pop().strip() - reasoning = DELIM.join(answer) - return Sample(input=input, target=target, metadata={"reasoning": reasoning.strip()}) - - -def sample_to_fewshot(sample): - return f"{sample.input}\n\nReasoning:\n" + f"{sample.metadata['reasoning']}\n\n" + f"ANSWER: {sample.target}" - - -def gsm8k_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"Question: {line['question']}\nAnswer:", - choices=[f" {line['answer']}"], - gold_index=0, - ) - - -gsm8k = LightevalTaskConfig( - name="gsm8k", - prompt_function=gsm8k_prompt, - sample_fields=record_to_sample, - sample_to_fewshot=sample_to_fewshot, - solver=[prompt_template(MATH_PROMPT_TEMPLATE), generate(cache=True)], - scorer=math_scorer(), - hf_repo="openai/gsm8k", - hf_subset="main", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select="random_sampling_from_train", - generation_size=256, - metrics=[ - Metrics.expr_gold_metric, - ], - stop_sequence=["Question:"], - version=0, -) - -TASKS_TABLE = [ - gsm8k, -] diff --git a/src/lighteval/tasks/tasks/gsm_plus.py b/src/lighteval/tasks/tasks/gsm_plus.py deleted file mode 100644 index 31c6409ce..000000000 --- a/src/lighteval/tasks/tasks/gsm_plus.py +++ /dev/null @@ -1,94 +0,0 @@ -""" -name: -Gsm Plus - -dataset: -qintongli/GSM-Plus - -abstract: -GSM-Plus is an adversarial extension of GSM8K that tests the robustness of LLMs' -mathematical reasoning by introducing varied perturbations to grade-school math -problems. - -languages: -english - -tags: -math, reasoning - -paper: -https://arxiv.org/abs/2402.19255 - -starred: -true -""" - -from inspect_ai.dataset import Sample -from inspect_ai.solver import generate, prompt_template - -from lighteval.metrics.metrics import Metrics, math_scorer -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -# setup for problem + instructions for providing answer -MATH_PROMPT_TEMPLATE = """ -Solve the following math problem step by step. The last line of your -response should be of the form "ANSWER: $ANSWER" (without quotes) -where $ANSWER is the answer to the problem. - -{prompt} - -Remember to put your answer on its own line at the end in the form -"ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to -the problem, and you do not need to use a \\boxed command. - -Reasoning: -""".strip() - - -def record_to_sample(record): - return Sample( - input=record["question"], - target=record["answer"], - metadata={"reasoning": record["solution"]}, - ) - - -def sample_to_fewshot(sample): - return f"{sample.input}\n\nReasoning:\n" + f"{sample.metadata['reasoning']}\n\n" + f"ANSWER: {sample.target}" - - -def gsm_plus_prompt(line, task_name: str = None): - if line["perturbation_type"] == "critical thinking": - return None - return Doc( - task_name=task_name, - query=f"Question: {line['question']}\n\nAnswer:", - choices=[line["answer"]], - gold_index=0, - ) - - -gsm_plus = LightevalTaskConfig( - name="gsm_plus", - prompt_function=gsm_plus_prompt, - sample_fields=record_to_sample, - sample_to_fewshot=sample_to_fewshot, - solver=[prompt_template(MATH_PROMPT_TEMPLATE), generate(cache=True)], - scorer=math_scorer(), - hf_repo="qintongli/GSM-Plus", - hf_subset="default", - hf_avail_splits=["test", "testmini"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.expr_gold_metric], - stop_sequence=None, - version=0, -) - -TASKS_TABLE = [ - gsm_plus, -] diff --git a/src/lighteval/tasks/tasks/headqa.py b/src/lighteval/tasks/tasks/headqa.py deleted file mode 100644 index 85c025c5f..000000000 --- a/src/lighteval/tasks/tasks/headqa.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -name: -Headqa - -dataset: -lighteval/headqa_harness - -abstract: -HEAD-QA is a multi-choice HEAlthcare Dataset. The questions come from exams to -access a specialized position in the Spanish healthcare system, and are -challenging even for highly specialized humans. They are designed by the -Ministerio de Sanidad, Consumo y Bienestar Social, who also provides direct -access to the exams of the last 5 years. - -languages: -english, spanish - -tags: -health, medical, multiple-choice, qa - -paper: -https://arxiv.org/abs/1906.04701 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def headqa_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"Question: {line['qtext']}\nAnswer:", - choices=[f" {answer['atext']}" for answer in line["answers"]], - gold_index=int(line["ra"]) - 1, - ) - - -headqa_en = LightevalTaskConfig( - name="headqa:en", - prompt_function=headqa_prompt, - hf_repo="lighteval/headqa_harness", - hf_subset="en", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=["\n"], - version=0, -) - - -headqa_es = LightevalTaskConfig( - name="headqa:es", - prompt_function=headqa_prompt, - hf_repo="lighteval/headqa_harness", - hf_subset="es", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - headqa_en, - headqa_es, -] diff --git a/src/lighteval/tasks/tasks/hellaswag.py b/src/lighteval/tasks/tasks/hellaswag.py deleted file mode 100644 index bb948f749..000000000 --- a/src/lighteval/tasks/tasks/hellaswag.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -name: -Hellaswag - -dataset: -Rowan/hellaswag - -abstract: -HellaSwag is a commonsense inference benchmark designed to challenge language -models with adversarially filtered multiple-choice questions. - -languages: -english - -tags: -multiple-choice, narrative, reasoning - -paper: -https://arxiv.org/abs/1905.07830 -""" - -from string import ascii_uppercase - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def hellaswag_prompt(line, task_name: str = None): - query = "The following are multiple choice questions (with answers) about common sense.\n\n" - query += f"Question: {line['activity_label']}: {line['ctx_a']} {line['ctx_b'].capitalize()}\n" - query += "".join([f"{key}. {choice}\n" for key, choice in zip(ascii_uppercase, line["endings"])]) - query += "Answer:" - - gold_ix = int(line["label"]) if line["label"] != "" else -1 - return Doc( - task_name=task_name, - query=query, - choices=[" " + i for i in ascii_uppercase[: len(line["endings"])]], - gold_index=gold_ix, - instruction="The following are multiple choice questions (with answers) about common sense.\n\n", - ) - - -hellaswag = LightevalTaskConfig( - name="hellaswag", - prompt_function=hellaswag_prompt, - hf_repo="Rowan/hellaswag", - hf_subset="default", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - hellaswag, -] diff --git a/src/lighteval/tasks/tasks/hle.py b/src/lighteval/tasks/tasks/hle.py deleted file mode 100644 index 43e375b3b..000000000 --- a/src/lighteval/tasks/tasks/hle.py +++ /dev/null @@ -1,254 +0,0 @@ -""" -name: -Humanity's Last Exam - -dataset: -cais/hle - -abstract: -Humanity's Last Exam (HLE) is a global collaborative effort, with questions from -nearly 1,000 subject expert contributors affiliated with over 500 institutions -across 50 countries - comprised mostly of professors, researchers, and graduate -degree holders. - -languages: -english - -tags: -qa, reasoning, general-knowledge - -paper: -https://arxiv.org/abs/2501.14249 - -starred: -true -""" - -import logging -import math -from typing import List, Literal - -import numpy as np -from aenum import extend_enum -from inspect_ai.dataset import Sample -from inspect_ai.scorer import model_graded_fact -from inspect_ai.solver import generate, system_message -from pydantic import BaseModel - -from lighteval.metrics.metrics import Metrics -from lighteval.metrics.metrics_sample import JudgeLLM -from lighteval.metrics.utils.metric_utils import CorpusLevelMetricGrouping -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc, SamplingMethod - - -logger = logging.getLogger(__name__) - - -class ExtractedAnswer(BaseModel): - extracted_final_answer: str - reasoning: str - correct: Literal["yes", "no"] - confidence: int - strict: Literal[True] # 100% reliability - - -# Adaptation from https://github.com/centerforaisafety/hle/blob/main/hle_eval/run_judge_results.py - - -def get_judge_prompt(question: str, answer: str, gold: str, **kwargs): - return [ - { - "role": "user", - "content": rf"""Judge whether the following [response] to [question] is correct or not based on the precise and unambiguous [correct_answer] below. - -[question]: {question} - -[response]: {answer} - -Your judgement must be in the format and criteria specified below: - -extracted_final_answer: The final exact answer extracted from the [response]. Put the extracted answer as 'None' if there is no exact, final answer to extract from the response. - -[correct_answer]: {gold} - -reasoning: Explain why the extracted_final_answer is correct or incorrect based on [correct_answer], focusing only on if there are meaningful differences between [correct_answer] and the extracted_final_answer. Do not comment on any background to the problem, do not attempt to solve the problem, do not argue for any answer different than [correct_answer], focus only on whether the answers match. - -correct: Answer 'yes' if extracted_final_answer matches the [correct_answer] given above, or is within a small margin of error for numerical problems. Answer 'no' otherwise, i.e. if there if there is any inconsistency, ambiguity, non-equivalency, or if the extracted answer is incorrect. - - -confidence: The extracted confidence score between 0|\%| and 100|\%| from [response]. Put 100 if there is no confidence score available.""", - }, - ] - - -def process_judge_response_hle(response: ExtractedAnswer | List[ExtractedAnswer]): - # todo: add support for batched responses - if isinstance(response, list): - response = response[0] - return { - # "correct_answer": correct_answer, - "model_answer": response.extracted_final_answer, - "reasoning": response.reasoning, - "correct": response.correct, - "confidence": response.confidence, - } - - -class JudgeLLMHLE(JudgeLLM): - def __init__(self): - super().__init__( - judge_model_name="gpt-4o-2024-08-06", - template=get_judge_prompt, - process_judge_response=process_judge_response_hle, - judge_backend="openai", - short_judge_name="hle_judge", - response_format=ExtractedAnswer, - ) - - def compute(self, sample_ids: list[str], responses: list, formatted_docs: list[Doc]) -> list[dict[str, float]]: - # If we are evaluating a multiturn task, we need to have specific field in the formatted doc - questions = [formatted_doc.specific["question"] for formatted_doc in formatted_docs] - golds = [formatted_doc.get_golds()[0] for formatted_doc in formatted_docs] - predictions = [response[0].result[0] for response in responses] - options = [None] * len(questions) - - score, _, _ = self.judge.evaluate_answer_batch(questions, predictions, options, golds) - - metrics = [] - for i in range(len(sample_ids)): - score[i]["correct_answer"] = golds[i] - metrics.append( - { - "accuracy": score[i], - "confidence_half_width": score[i], - "calibration_error": score[i], - } - ) - - return metrics - - def compute_corpus(self, scores: List[dict]): - n = len(scores) - - correct = [] - confidence = [] - for score in scores: - correct.append("yes" in score["correct"]) - confidence.append(score["confidence"]) - - correct = np.array(correct) - confidence = np.array(confidence) - - # sometimes model collapses on same questions - if len(correct) != n: - print(f"Available predictions: {len(correct)} | Total questions: {n}") - - accuracy = round(100 * sum(correct) / n, 2) - # Wald estimator, 95% confidence interval - confidence_half_width = round(1.96 * math.sqrt(accuracy * (100 - accuracy) / n), 2) - calibration_error = round(calib_err(confidence, correct, p="2", beta=100), 2) - - return { - "accuracy": accuracy, - "confidence_half_width": confidence_half_width, - "calibration_error": calibration_error, - } - - -# source: https://github.com/hendrycks/outlier-exposure/blob/master/utils/calibration_tools.py -def calib_err(confidence, correct, p="2", beta=100): - # beta is target bin size - idxs = np.argsort(confidence) - confidence = confidence[idxs] - correct = correct[idxs] - bins = [[i * beta, (i + 1) * beta] for i in range(len(confidence) // beta)] - if len(bins) == 0: - logger.warning("Error when computing the bins for calibration error") - return -1 - - bins[-1] = [bins[-1][0], len(confidence)] - - cerr = 0 - total_examples = len(confidence) - for i in range(len(bins) - 1): - bin_confidence = confidence[bins[i][0] : bins[i][1]] - bin_correct = correct[bins[i][0] : bins[i][1]] - num_examples_in_bin = len(bin_confidence) - - if num_examples_in_bin > 0: - difference = np.abs(np.nanmean(bin_confidence) - np.nanmean(bin_correct)) - - if p == "2": - cerr += num_examples_in_bin / total_examples * np.square(difference) - elif p == "1": - cerr += num_examples_in_bin / total_examples * difference - elif p == "infty" or p == "infinity" or p == "max": - cerr = np.maximum(cerr, difference) - else: - assert False, "p must be '1', '2', or 'infty'" - - if p == "2": - cerr = np.sqrt(cerr) - - return cerr - - -SYSTEM_MESSAGE = """ -Your response should be in the following format:\nExplanation: {your explanation for your answer choice}\nAnswer: {your chosen answer}\nConfidence: {your confidence score between 0% and 100% for your answer} -""".strip() - - -def hle_text_only(line, task_name: str = None): - if line["image"] not in [None, ""]: - return None - - return Doc( - task_name=task_name, - query=line["question"], - choices=[line["answer"]], - gold_index=0, - instruction=SYSTEM_MESSAGE, - specific={"question": line["question"]}, - ) - - -hle_metrics = CorpusLevelMetricGrouping( - metric_name=["accuracy", "confidence_half_width", "calibration_error"], - higher_is_better=dict.fromkeys(["accuracy", "confidence_half_width", "calibration_error"], True), - category=SamplingMethod.GENERATIVE, - sample_level_fn=JudgeLLMHLE(), - corpus_level_fn=JudgeLLMHLE(), -) -extend_enum(Metrics, "hle_metrics", hle_metrics) - - -def record_to_sample(record): - return Sample( - input=record["question"], - target=record["answer"], - metadata={"is_image_question": record["image"] not in [None, ""]}, - ) - - -hle = LightevalTaskConfig( - name="hle", - prompt_function=hle_text_only, - hf_repo="cais/hle", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8192, - metrics=[Metrics.exact_match, Metrics.hle_metrics], - stop_sequence=[], - version=0, - sample_fields=record_to_sample, - solver=[system_message(SYSTEM_MESSAGE), generate(cache=True)], - scorer=model_graded_fact(), - filter=lambda x: not x.metadata["is_image_question"], -) - - -TASKS_TABLE = [hle] diff --git a/src/lighteval/tasks/tasks/ifbench/evaluation_lib.py b/src/lighteval/tasks/tasks/ifbench/evaluation_lib.py deleted file mode 100644 index 2c4b761e8..000000000 --- a/src/lighteval/tasks/tasks/ifbench/evaluation_lib.py +++ /dev/null @@ -1,216 +0,0 @@ -# coding=utf-8 -# Copyright 2025 The Google Research Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Binary of evaluating instruction following. See README.md.""" - -import collections -import dataclasses -import json -from typing import Dict, Optional, Union - -import lighteval.tasks.tasks.ifbench.instructions_registry as instructions_registry - - -@dataclasses.dataclass -class InputExample: - key: int - instruction_id_list: list[str] - prompt: str - kwargs: list[Dict[str, Optional[Union[str, int]]]] - - -@dataclasses.dataclass -class OutputExample: - instruction_id_list: list[str] - prompt: str - response: str - follow_all_instructions: bool - follow_instruction_list: list[bool] - - -def read_prompt_list(input_jsonl_filename): - """Read inputs from jsonl.""" - inputs = [] - with open(input_jsonl_filename, "r") as f: - for line in f: - example = json.loads(line) - inputs.append( - InputExample( - key=example["key"], - instruction_id_list=example["instruction_id_list"], - prompt=example["prompt"], - kwargs=example["kwargs"], - ) - ) - return inputs - - -def write_outputs(output_jsonl_filename, outputs): - """Writes outputs to jsonl.""" - assert outputs - with open(output_jsonl_filename, "w") as f: - for o in outputs: - f.write( - json.dumps( - { - attr_name: o.__getattribute__(attr_name) - for attr_name in [name for name in dir(o) if not name.startswith("_")] - } - ) - ) - f.write("\n") - - -def test_instruction_following_strict( - inp, - prompt_to_response, -): - """Tests response to see if instrutions are followed.""" - response = prompt_to_response[inp.prompt] - instruction_list = inp.instruction_id_list - is_following_list = [] - - for index, instruction_id in enumerate(instruction_list): - instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id] - instruction = instruction_cls(instruction_id) - inp.kwargs[index] = {key: value for key, value in inp.kwargs[index].items() if value is not None} - instruction.build_description(**inp.kwargs[index]) - args = instruction.get_instruction_args() - if args and "prompt" in args: - instruction.build_description(prompt=inp.prompt) - - if response.strip() and instruction.check_following(response): - is_following_list.append(True) - else: - is_following_list.append(False) - - return OutputExample( - instruction_id_list=inp.instruction_id_list, - prompt=inp.prompt, - response=response, - follow_all_instructions=all(is_following_list), - follow_instruction_list=is_following_list, - ) - - -def test_instruction_following_loose( - inp, - prompt_to_response, -): - """Tests response for an upper bound for following instructions.""" - response = prompt_to_response[inp.prompt] - r = response.split("\n") - response_remove_first = "\n".join(r[1:]).strip() - response_remove_last = "\n".join(r[:-1]).strip() - response_remove_both = "\n".join(r[1:-1]).strip() - revised_response = response.replace("*", "") - revised_response_remove_first = response_remove_first.replace("*", "") - revised_response_remove_last = response_remove_last.replace("*", "") - revised_response_remove_both = response_remove_both.replace("*", "") - all_responses = [ - response, - revised_response, - response_remove_first, - response_remove_last, - response_remove_both, - revised_response_remove_first, - revised_response_remove_last, - revised_response_remove_both, - ] - instruction_list = inp.instruction_id_list - is_following_list = [] - - for index, instruction_id in enumerate(instruction_list): - instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id] - instruction = instruction_cls(instruction_id) - - instruction.build_description(**inp.kwargs[index]) - args = instruction.get_instruction_args() - if args and "prompt" in args: - instruction.build_description(prompt=inp.prompt) - - is_following = False - for r in all_responses: - if r.strip() and instruction.check_following(r): - is_following = True - break - - is_following_list.append(is_following) - - return OutputExample( - instruction_id_list=inp.instruction_id_list, - prompt=inp.prompt, - response=response, - follow_all_instructions=all(is_following_list), - follow_instruction_list=is_following_list, - ) - - -def read_prompt_to_response_dict(input_jsonl_filename): - """Creates dictionary matching prompt and response.""" - return_dict = {} - with open(input_jsonl_filename, "r") as file: - for line in file: - example = json.loads(line) - return_dict[example["prompt"]] = example["response"] - return return_dict - - -def print_report(outputs): - """Prints a report on accuracy scores.""" - - prompt_total = 0 - prompt_correct = 0 - instruction_total = 0 - instruction_correct = 0 - - tier0_total = collections.defaultdict(int) - tier0_correct = collections.defaultdict(int) - - tier1_total = collections.defaultdict(int) - tier1_correct = collections.defaultdict(int) - - for example in outputs: - follow_instruction_list = example.follow_instruction_list - instruction_id_list = example.instruction_id_list - - prompt_total += 1 - if all(follow_instruction_list): - prompt_correct += 1 - - instruction_total += len(instruction_id_list) - instruction_correct += sum(follow_instruction_list) - - for instruction_id, followed_or_not in zip(instruction_id_list, follow_instruction_list): - instruction_id = instruction_id.split(":")[0] - tier0_total[instruction_id] += 1 - if followed_or_not: - tier0_correct[instruction_id] += 1 - - for instruction_id, followed_or_not in zip(instruction_id_list, follow_instruction_list): - tier1_total[instruction_id] += 1 - if followed_or_not: - tier1_correct[instruction_id] += 1 - - print(f"prompt-level: {prompt_correct / prompt_total}") - print(f"instruction-level: {instruction_correct / instruction_total}") - print() - for instruction_id in sorted(tier0_total.keys()): - accuracy = tier0_correct[instruction_id] / tier0_total[instruction_id] - print(f"{instruction_id} {accuracy}") - print() - for instruction_id in sorted(tier1_total.keys()): - accuracy = tier1_correct[instruction_id] / tier1_total[instruction_id] - print(f"{instruction_id} {accuracy}") diff --git a/src/lighteval/tasks/tasks/ifbench/instructions.py b/src/lighteval/tasks/tasks/ifbench/instructions.py deleted file mode 100644 index bfdd10067..000000000 --- a/src/lighteval/tasks/tasks/ifbench/instructions.py +++ /dev/null @@ -1,2339 +0,0 @@ -# Copyright 2025 Allen Institute for AI. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Library of instructions.""" - -import csv -import io -import logging -import random -import re -import string -import unicodedata -from collections import Counter - -import nltk - -from lighteval.utils.imports import is_package_available, requires - - -if is_package_available("syllapy"): - import syllapy - -if is_package_available("spacy"): - import spacy - -if is_package_available("emoji"): - import emoji - -import lighteval.tasks.tasks.ifeval.instructions_utils as instructions_util - - -logger = logging.getLogger(__name__) - -# The number of words in the response. -_NUM_WORDS_LOWER_LIMIT = 100 -_NUM_WORDS_UPPER_LIMIT = 500 - -# The number of numbers. -_NUM_NUMBERS = 6 - -# Period length for periodic words. -_NUM_WORD_CYCLE = 30 - -# Maximum number of times a word can be repeated. -_MAX_REPEATS = 5 - -# Which sentence must contain a keyword. -_NUM_KEYWORD_SENTENCE = 20 - -# Minimum number of pronouns. -_NUM_PRONOUNS = 25 - -# The size of increment for lengths. -_NUM_INCREMENT = 5 - -# The number of coordinating conjunctions. -_NUM_CONJUNCTIONS = 6 - -RESOURCES_DOWNLOADED: bool = False - - -@requires("syllapy", "spacy") -class Instruction: - """An instruction template.""" - - def __init__(self, instruction_id): - self.id = instruction_id - - global RESOURCES_DOWNLOADED - if not RESOURCES_DOWNLOADED: - nltk.download("punkt_tab") - nltk.download("averaged_perceptron_tagger_eng") - - try: - spacy.load("en_core_web_sm") - except OSError: - logger.info("Downloading the spacy en_core_web_sm model\n(don't worry, this will only happen once)") - from spacy.cli import download - - download("en_core_web_sm") - RESOURCES_DOWNLOADED = True - - def build_description(self, **kwargs): - raise NotImplementedError("`build_description` not implemented.") - - def get_instruction_args(self): - raise NotImplementedError("`get_instruction_args` not implemented.") - - def get_instruction_args_keys(self): - raise NotImplementedError("`get_instruction_args_keys` not implemented.") - - def check_following(self, value): - raise NotImplementedError("`check_following` not implemented.") - - -class WordCountRangeChecker(Instruction): - """Word Count Range: The response must contain between X and Y words.""" - - def build_description(self, *, min_words=None, max_words=None): - """Build the instruction description. - - Args: - min_words: An integer specifying the minimum number of words contained in the response. - max_words: An integer specifying the maximum number of words contained in the response. - - Returns: - A string representing the instruction description. - """ - self._min_words = min_words - self._max_words = max_words - - if self._min_words is None or self._min_words < 0: - self._min_words = random.randint(_NUM_WORDS_LOWER_LIMIT, _NUM_WORDS_UPPER_LIMIT) - - # Make the range small - if self._max_words is None or self._max_words < 0: - self._max_words = self._min_words + random.randint(int(self._min_words * 0.05), int(self._min_words * 0.1)) - - self._description_pattern = "The response must contain between {min_words} and {max_words} words." - - return self._description_pattern.format(min_words=self._min_words, max_words=self._max_words) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"min_words": self._min_words, "max_words": self._max_words} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["min_words", "max_words"] - - def check_following(self, value): - """Checks if the response contains the expected number of words.""" - num_words = instructions_util.count_words(value) - return self._min_words <= num_words <= self._max_words - - -class UniqueWordCountChecker(Instruction): - """Unique Word Count: The response must contain X unique words.""" - - def build_description(self, *, N=None): - """Build the instruction description. - - Args: - N: An integer specifying the number of unique words contained in the response. - - Returns: - A string representing the instruction description. - """ - self._num_unique_words = N - - if self._num_unique_words is None or self._num_unique_words < 0: - self._num_unique_words = random.randint(_NUM_WORDS_LOWER_LIMIT, _NUM_WORDS_UPPER_LIMIT) - - self._description_pattern = "Use at least {N} unique words in the response." - - return self._description_pattern.format(N=self._num_unique_words) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"N": self._num_unique_words} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["N"] - - def check_following(self, value): - """Checks if the response contains the expected number of unique words.""" - words = value.lower().split() - unique_words = set() - for word in words: - unique_words.add(word.strip("".join(string.punctuation) + " ")) - # Convert to set to get unique words - return len(unique_words) >= self._num_unique_words - - -class StopWordPercentageChecker(Instruction): - """Ensure that stop words constitute no more than {percentage}% of the total words in your response.""" - - def build_description(self, *, percentage=None): - """Build the instruction description. - - Args: - percentage: An integer specifying the percentage of stop words that are allowed in the response. - - Returns: - A string representing the instruction description. - """ - self._percentage = percentage - - if self._percentage is None or self._percentage < 0: - self._percentage = random.randint(1, 100) - - self._description_pattern = ( - "Ensure that stop words constitute no more than {percentage}% of the total words in your response." - ) - - return self._description_pattern.format(percentage=self._percentage) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"percentage": self._percentage} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["percentage"] - - def check_following(self, value): - """Checks if the response contains the expected percentage of stop words.""" - num_words = instructions_util.count_words(value) - num_stopwords = instructions_util.count_stopwords(value) - stopword_percentage = (num_stopwords / num_words) * 100 - return stopword_percentage <= self._percentage - - -class SentTypeRatioChecker(Instruction): - """Maintain a 2:1 ratio of declarative to interrogative sentences.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Maintain a 2:1 ratio of declarative to interrogative sentences." - return self._description_pattern - - def get_instruction_args(self): - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response contains the expected ratio of declarative to interrogative sentences.""" - # Split the text into sentences - sentences = instructions_util.split_into_sentences(value) - # Count the number of declarative and interrogative sentences - declarative_count = sum(1 for sentence in sentences if sentence.endswith(".")) - interrogative_count = sum(1 for sentence in sentences if sentence.endswith("?")) - # Check if the ratio is 2:1 - return declarative_count == 2 * interrogative_count - - -class SentBalanceChecker(Instruction): - """Ensure that the ratio of sentence types (declarative, interrogative, exclamatory) is balanced.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = ( - "Ensure that the ratio of sentence types (declarative, interrogative, exclamatory) is balanced." - ) - return self._description_pattern - - def get_instruction_args(self): - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response contains a balanced ratio of sentence types.""" - # Split the text into sentences - sentences = instructions_util.split_into_sentences(value) - # Count the number of each sentence type - declarative_count = sum(1 for sentence in sentences if sentence.endswith(".")) - interrogative_count = sum(1 for sentence in sentences if sentence.endswith("?")) - exclamatory_count = sum(1 for sentence in sentences if sentence.endswith("!")) - # Check if the ratio of sentence types is balanced - return declarative_count == interrogative_count == exclamatory_count - - -class ConjunctionCountChecker(Instruction): - """Use at least {small_n} different coordinating conjunctions in the response.""" - - def build_description(self, *, small_n=None): - """Build the instruction description. - - Args: - small_n: An integer specifying the number of different coordinating conjunctions contained in the response. - - Returns: - A string representing the instruction description. - """ - self._num_conjunctions = small_n - - if self._num_conjunctions is None or self._num_conjunctions < 0: - self._num_conjunctions = random.randint(2, _NUM_CONJUNCTIONS) - - self._description_pattern = "Use at least {small_n} different coordinating conjunctions in the response." - - return self._description_pattern.format(small_n=self._num_conjunctions) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"small_n": self._num_conjunctions} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["small_n"] - - def check_following(self, value): - """Checks if the response contains the expected number of different coordinating conjunctions.""" - # Split the text into words - words = value.split() - # Count the number of coordinating conjunctions - conjunctions = [ - word - for word in words - if word.strip("".join(string.punctuation) + " ").lower() in ["and", "but", "for", "nor", "or", "so", "yet"] - ] - unique_conjunctions = set(conjunctions) - return len(unique_conjunctions) >= self._num_conjunctions - - -class PersonNameCountChecker(Instruction): - """Mention at least {N} different person names in the response.""" - - def build_description(self, *, N=None): - """Build the instruction description. - - Args: - N: An integer specifying the minimum number of unique person names contained in the response. - - Returns: - A string representing the instruction description. - """ - self._num_person_names = N - - if self._num_person_names is None or self._num_person_names < 0: - self._num_person_names = random.randint(1, 50) - - self._description_pattern = "Mention at least {N} different person names in the response, from this list of person names: Emma, Liam, Sophia, Jackson, Olivia, Noah, Ava, Lucas, Isabella, Mason, Mia, Ethan, Charlotte, Alexander, Amelia, Benjamin, Harper, Leo, Zoe, Daniel, Chloe, Samuel, Lily, Matthew, Grace, Owen, Abigail, Gabriel, Ella, Jacob, Scarlett, Nathan, Victoria, Elijah, Layla, Nicholas, Audrey, David, Hannah, Christopher, Penelope, Thomas, Nora, Andrew, Aria, Joseph, Claire, Ryan, Stella, Jonathan ." - return self._description_pattern.format(N=self._num_person_names) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"N": self._num_person_names} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["N"] - - def check_following(self, value): - """Checks if the response contains at least the expected number of unique person names.""" - person_name_list = [ - "Emma", - "Liam", - "Sophia", - "Jackson", - "Olivia", - "Noah", - "Ava", - "Lucas", - "Isabella", - "Mason", - "Mia", - "Ethan", - "Charlotte", - "Alexander", - "Amelia", - "Benjamin", - "Harper", - "Leo", - "Zoe", - "Daniel", - "Chloe", - "Samuel", - "Lily", - "Matthew", - "Grace", - "Owen", - "Abigail", - "Gabriel", - "Ella", - "Jacob", - "Scarlett", - "Nathan", - "Victoria", - "Elijah", - "Layla", - "Nicholas", - "Audrey", - "David", - "Hannah", - "Christopher", - "Penelope", - "Thomas", - "Nora", - "Andrew", - "Aria", - "Joseph", - "Claire", - "Ryan", - "Stella", - "Jonathan", - ] - # Extract the named entities - person_names = [] - for name in person_name_list: - if name in value: - person_names.append(name) - unique_person_names = set(person_names) - - return len(unique_person_names) >= self._num_person_names - - -class NGramOverlapChecker(Instruction): - """Maintain a trigram overlap of {percentage}% (±2%) with the provided reference text.""" - - def build_description(self, *, reference_text=None, percentage=None): - """Build the instruction description. - - Args: - reference_text: A string representing the reference text. - percentage: An integer specifying the percent trigram overlap - to maintain in the response. - - Returns: - A string representing the instruction description. - """ - self._reference_text = reference_text - self._percentage = percentage - if self._percentage is None or self._percentage < 0: - self._percentage = random.randint(1, 100) - - self._description_pattern = ( - "Maintain a trigram overlap of {percentage}% (±2%) with the provided reference text." - ) - return self._description_pattern.format(percentage=self._percentage) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"reference_text": self._reference_text, "percentage": self._percentage} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["reference_text", "percentage"] - - def check_following(self, value): - """Checks if the response maintains a trigram overlap with the reference text within 2% of {percent}.""" - n = 3 - ngrams = set(nltk.ngrams(value, n)) - ref_ngrams = set(nltk.ngrams(self._reference_text, n)) - overlap = len(ngrams.intersection(ref_ngrams)) / len(ngrams) - return self._percentage - 2 <= overlap * 100 <= self._percentage + 2 - - -class NumbersCountChecker(Instruction): - """Include exactly {N} numbers in the response.""" - - def build_description(self, *, N=None): - """Build the instruction description. - - Args: - N: An integer specifying the exact number of numbers - that is required to appear in the response. - - Returns: - A string representing the instruction description. - """ - self._count_numbers = N - if self._count_numbers is None or self._count_numbers < 0: - self._count_numbers = random.randint(1, _NUM_NUMBERS) - - self._description_pattern = "Include exactly {N} numbers in the response." - return self._description_pattern.format(N=self._count_numbers) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"N": self._count_numbers} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["N"] - - def check_following(self, value): - """Checks if the response includes exactly {N} numbers.""" - # Strip punctuation to handle decimals and commas in numbers correctly - value = value.translate(str.maketrans("", "", string.punctuation)) - numbers = re.findall(r"\d+", value) - return len(numbers) == self._count_numbers - - -class AlphabetLoopChecker(Instruction): - """Each word must start with the next letter of the alphabet, looping back to 'A' after 'Z'.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = ( - "Each word must start with the next letter of the alphabet, looping back to 'A' after 'Z'." - ) - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if each word of the response starts with the next letter of the alphabet.""" - value = value.translate(str.maketrans("", "", string.punctuation)) - words = value.strip("".join(string.punctuation) + " ").split() - alphabet = string.ascii_lowercase - correct_letter = words[0][0].lower() - if correct_letter not in alphabet: # numbers are fails - return False - for word in words[1:]: - word = word.strip("".join(string.punctuation) + " ").lower() - if not word: - continue - correct_letter = alphabet[(alphabet.index(correct_letter) + 1) % 26] - if word[0] != correct_letter: - return False - return True - - -class SingleVowelParagraphChecker(Instruction): - """Write a paragraph using words that contain only three type of vowels.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Write a paragraph using words that contain only three types of vowels." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if no more than three types of vowels are used in the response and the response is only 1 paragraph.""" - paragraphs = value.strip().split("\n") - if len(paragraphs) != 1: - return False - paragraph = paragraphs[0].lower() - - vowels = set("aeiou") - paragraph_vowels = {char for char in paragraph if char in vowels} - return len(paragraph_vowels) <= 3 - - -class ConsonantClusterChecker(Instruction): - """Ensure each word in your response has at least one consonant cluster (two or more consonants together).""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = ( - "Ensure each word in your response has at least one consonant cluster (two or more consonants together)." - ) - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if each word in the response includes at least one consonant cluster.""" - words = value.lower().strip().split() - consonants = set("bcdfghjklmnpqrstvwxyz") - for word in words: - cluster = False - for i in range(len(word) - 1): - if word[i] in consonants and word[i + 1] in consonants: - cluster = True - break - if not cluster: - return False - return True - - -class IncrementingAlliterationChecker(Instruction): - """Each sentence must have a longer sequence of consecutive alliterative words than the previous one.""" - - def build_description(self): - """Build the instruction description.""" - - self._description_pattern = ( - "Each sentence must have a longer sequence of consecutive alliterative words than the previous one." - ) - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if each sentence in the response has more alliterative words (determined by common first letter) than the previous sentence.""" - sentences = instructions_util.split_into_sentences(value) - prev_alliteration = -1 - for sentence in sentences: - words = sentence.lower().split() - alliteration = 0 - prev_alliterative = False - new_words = [] - for word in words: - clean = word.lstrip("".join(string.punctuation) + " ") - if clean: - new_words.append(clean) - for i in range(len(new_words) - 1): - if new_words[i][0] == new_words[i + 1][0]: - if prev_alliterative: - alliteration += 1 - else: - alliteration += 2 - prev_alliterative = True - else: - prev_alliterative = False - if alliteration <= prev_alliteration: - return False - prev_alliteration = alliteration - return True - - -class PalindromeChecker(Instruction): - """Include at least 10 single-word palindromes, each at least 5 characters long.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Include at least 10 single-word palindromes, each at least 5 characters long." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response includes at least 10 single-word palindromes of length at least 5.""" - value = value.translate(str.maketrans("", "", string.punctuation)) - words = value.lower().split() - palindromes = [word for word in words if word == word[::-1] and len(word) >= 5] - return len(palindromes) >= 10 - - -class PunctuationCoverChecker(Instruction): - """Use every standard punctuation mark at least once, including semicolons, colons, and the interrobang (?!).""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Use every standard punctuation mark at least once, including semicolons, colons, and the interrobang (?!)." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response includes every standard punctuation mark at least once, including the interrobang (?!).""" - punctuation = {".", ",", "!", "?", ";", ":"} - if not ("!?" in value or "?!" in value or "‽" in value): - return False - new_value = value.replace("?!", "", 1) - if len(new_value) == len(value): - new_value = value.replace("!?", "", 1) - for char in new_value: - if char in punctuation: - punctuation.remove(char) - return not punctuation - - -class NestedParenthesesChecker(Instruction): - """Nest parentheses (and [brackets {and braces}]) at least 5 levels deep.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Nest parentheses (and [brackets {and braces}]) at least 5 levels deep." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response includes a correctly closed set of at least 5 nested brackets.""" - levels = [] - min_levels = 5 - max_depth = 0 - - for char in value: - if char in "([{": - levels.append(char) - if len(levels) > max_depth: - max_depth = len(levels) - elif char in ")]}": - if levels and ( - (levels[-1] == "(" and char == ")") - or (levels[-1] == "[" and char == "]") - or (levels[-1] == "{" and char == "}") - ): - levels.pop() - # Check if we just closed a group that reached 5+ depth - if max_depth >= min_levels and len(levels) < max_depth: - return True - else: - # Mismatch — reset - levels = [] - max_depth = 0 - - return False - - -class NestedQuotesChecker(Instruction): - """Include quotes within quotes within quotes, at least 3 levels deep, alternating between double quotes and single quotes.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Include quotes within quotes within quotes, at least 3 levels deep, alternating between double quotes and single quotes." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response includes nested quotes to at least 3 levels - alternating between " and ' starting with either character.""" - levels = [] - min_levels = 3 - reached_depth = 0 - current_depth = 0 - for char in value: - if len(levels) != 0 and char == levels[-1]: - levels.pop() - current_depth -= 1 - if reached_depth - current_depth >= min_levels: - return True - elif char == '"' or char == "'": - levels.append(char) - current_depth += 1 - if current_depth > reached_depth: - reached_depth = current_depth - return False - - -class PrimeLengthsChecker(Instruction): - """Use only words with lengths that are prime numbers.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Use only words with lengths that are prime numbers." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response only includes words with prime length.""" - value = value.translate(str.maketrans("", "", string.punctuation)) - words = value.split() - primes = {2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97} - for word in words: - if len(word) not in primes: - return False - return True - - -class OptionsResponseChecker(Instruction): - """Answer with one of the following options: {options}. Do not give any explanation.""" - - def build_description(self, *, options=None): - """Build the instruction description. - - Args: - options: A string specifying the permitted options for - the response. - - Returns: - A string representing the instruction description. - """ - # Options string may be: yes/no/maybe, I know or I don't know, a), b), c), d) - # Can be separated by "/", "or", "," - options_bank = ["yes/no/maybe", "I know or I don't know", "a), b), c), d)"] - if options is None: - options = random.choice(options_bank) - - # Be more strict about format for multiple choice letters than for text options - self._strict = False - if re.match(r"\W*[aA]\W*[bB]\W*[cC]\W*", options) is not None: - self._strict = True - if "/" in options: - separator = "/" - elif "or" in options: - separator = "or" - else: - separator = "," - self._options = [option.strip() for option in options.split(separator)] - self._options_text = options # in text, shouldn't be formatted as a list - self._description_pattern = "Answer with one of the following options: {options}. Do not give any explanation." - return self._description_pattern.format(options=self._options_text) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"options": self._options_text} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["options"] - - def check_following(self, value): - """Checks if the response is exactly one of {options}.""" - if self._strict: - return value in self._options - value = value.strip("".join(string.punctuation) + " ").lower() - for option in self._options: - if option.strip("".join(string.punctuation) + " ").lower() == value: - return True - return False - - -class NewLineWordsChecker(Instruction): - """Write each word on a new line.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Write each word on a new line." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response has each word on a new line.""" - value = value.translate(str.maketrans("", "", string.punctuation)) - lines = value.strip().split("\n") - while "" in lines: - lines.remove("") - return len(lines) == len(value.strip().split()) - - -class EmojiSentenceChecker(Instruction): - """Please use an emoji at the end of every sentence.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Please use an emoji at the end of every sentence." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response includes an emoji at the end of every sentence.""" - - sentences = instructions_util.split_into_sentences(value) - for i, sentence in enumerate(sentences): - stripped = sentence.translate(str.maketrans("", "", string.punctuation)).strip() - last_char = stripped[-1] - # because blank spaces are treated oddly - second_last_char = stripped[-2] if len(stripped) > 1 else stripped[-1] - if not emoji.is_emoji(last_char) and not emoji.is_emoji(second_last_char): - if i < len(sentences) - 1: - stripped = sentences[i + 1].translate(str.maketrans("", "", string.punctuation)).strip() - first_char = stripped[0] - if not emoji.is_emoji(first_char): - return False - else: - return False - return True - - -class CharacterCountUniqueWordsChecker(Instruction): - """Respond with three sentences, all containing the same number of characters but using all different words.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = ( - "Respond with three sentences, all containing the same number of characters but using all different words." - ) - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response has exactly 3 sentences containing the same number of characters but different words.""" - sentences = instructions_util.split_into_sentences(value) - if len(sentences) != 3: - return False - char_count = len(sentences[0].strip()) - for sentence in sentences: - if len(sentence.strip()) != char_count: - return False - return True - - -class NthWordJapaneseChecker(Instruction): - """Every {N}th word of your response must be in Japanese.""" - - def build_description(self, *, N=None): - """Build the instruction description. - - Args: - N: An integer specifying the cycle length for - Japanese words to appear in the response. - - Returns: - A string representing the instruction description. - """ - self._japanese_position = N - if self._japanese_position is None or self._japanese_position < 0: - self._japanese_position = random.randint(1, _NUM_WORD_CYCLE) - - self._description_pattern = "Every {N}th word of your response must be in Japanese." - if N % 10 == 1: - self._description_pattern = "Every {N}st of your response must be in Japanese." - if N % 10 == 2: - self._description_pattern = "Every {N}nd of your response must be in Japanese." - elif N % 10 == 3: - self._description_pattern = "Every {N}rd of your response must be in Japanese." - return self._description_pattern.format(N=self._japanese_position) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"N": self._japanese_position} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["N"] - - def check_following(self, value): - """Checks if every {N}th word of the response is in Japanese.""" - - def is_japanese(text): - """ - Checks if a string contains Japanese characters (Hiragana, Katakana, or Kanji). - - Args: - text: The string to check. - - Returns: - True if the string contains Japanese characters, False otherwise. - """ - japanese_pattern = re.compile(r"[\u3040-\u30ff\u4e00-\u9fff]") - return bool(japanese_pattern.search(text)) - - words = value.split() - for i, word in enumerate(words): - word = word.strip("".join(string.punctuation) + " ") - if (i + 1) % self._japanese_position == 0 and word and not word.isdigit(): - if not is_japanese(word): - return False - return True - - -class StartWithVerbChecker(Instruction): - """The response must start with a verb.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "The response must start with a verb." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response starts with a verb.""" - text = nltk.word_tokenize(value) - return len(text) > 0 and len(nltk.pos_tag(text)) > 0 and "VB" in nltk.pos_tag(text)[0][1] - - -class LimitedWordRepeatChecker(Instruction): - """The response should not repeat any word more than {small_n} times.""" - - def build_description(self, *, small_n=None): - """Build the instruction description. - - Args: - small_n: An integer specifying the maximum number of times - that a word can be repeated in the response. - - Returns: - A string representing the instruction description. - """ - self._max_repeats = small_n - if self._max_repeats is None or self._max_repeats < 0: - self._max_repeats = random.randint(1, _MAX_REPEATS) - - self._description_pattern = "The response should not repeat any word more than {small_n} times." - return self._description_pattern.format(small_n=self._max_repeats) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"small_n": self._max_repeats} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["small_n"] - - def check_following(self, value): - """Checks if the response repeats any word more than {small_n} times.""" - words = value.lower().translate(str.maketrans("", "", string.punctuation)).split() - word_count = Counter(words) - for word, count in word_count.items(): - if count > self._max_repeats: - return False - return True - - -class IncludeKeywordChecker(Instruction): - """The response must include keyword {word} in the {N}-th sentence.""" - - def build_description(self, *, word=None, N=None): - """Build the instruction description. - - Args: - word: A string specifying the keyword that is - required to appear in the response. - N: An integer specifying which sentence of the - response is required to have the keyword. - - Returns: - A string representing the instruction description. - """ - - if not word: - self._keyword = instructions_util.generate_keywords(num_keywords=1)[0] - else: - self._keyword = word - self._keyword_position = N - if self._keyword_position is None or self._keyword_position < 0: - self._keyword_position = random.randint(1, _NUM_KEYWORD_SENTENCE) - - self._description_pattern = 'The response must include keyword "{word}" in the {N}-th sentence.' - return self._description_pattern.format(word=self._keyword, N=self._keyword_position) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"word": self._keyword, "N": self._keyword_position} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["word", "N"] - - def check_following(self, value): - """Checks if the {N}th sentence of the response includes keyword {word}.""" - sentences = instructions_util.split_into_sentences(value) - if len(sentences) < self._keyword_position: - return False - return self._keyword.lower() in sentences[int(self._keyword_position - 1)].lower() - - -class PronounCountChecker(Instruction): - """The response should include at least {N} pronouns.""" - - def build_description(self, *, N=None): - """Build the instruction description. - - Args: - N: An integer specifying the minimum number of pronouns - that is required to appear in the response. - - Returns: - A string representing the instruction description. - """ - self._num_pronouns = N - if self._num_pronouns is None or self._num_pronouns < 0: - self._num_pronouns = random.randint(1, _NUM_PRONOUNS) - - self._description_pattern = "The response should include at least {N} pronouns." - return self._description_pattern.format(N=self._num_pronouns) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"N": self._num_pronouns} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["N"] - - def check_following(self, value): - """Checks if the response includes at least {N} pronouns.""" - pronouns = { - "i", - "me", - "my", - "mine", - "myself", - "we", - "us", - "our", - "ours", - "ourselves", - "you", - "your", - "yours", - "yourself", - "yourselves", - "he", - "him", - "his", - "himself", - "she", - "her", - "hers", - "herself", - "it", - "its", - "itself", - "they", - "them", - "their", - "theirs", - "themselves", - } - value = value.replace( - "/", " " - ) # to correctly count pronoun sets like she/her/hers, a common use case of pronouns - value = value.lower().translate(str.maketrans("", "", string.punctuation)) - words = value.split() - pronoun_count = sum(1 for word in words if word in pronouns) - return pronoun_count >= self._num_pronouns - - -class AlternateParitySyllablesChecker(Instruction): - """Alternate between words with odd and even numbers of syllables.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Alternate between words with odd and even numbers of syllables." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response alternates between words with odd and even numbers of syllables.""" - words = value.translate(str.maketrans("", "", string.punctuation)).lower().split() - syllables = [syllapy.count(word) % 2 for word in words if word.strip()] - return all(syllables[i] != syllables[i + 1] for i in range(len(syllables) - 1)) - - -class LastWordFirstNextChecker(Instruction): - """The last word of each sentence must become the first word of the next sentence.""" - - def build_description(self): - """Build the instruction description.""" - - self._description_pattern = "The last word of each sentence must become the first word of the next sentence." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the last word of each sentence in the response is the first word of the next sentence.""" - sentences = instructions_util.split_into_sentences(value) - for i in range(len(sentences) - 1): - last_word = sentences[i].rstrip("".join(string.punctuation) + " ").split()[-1] - first_word = sentences[i + 1].lstrip("".join(string.punctuation) + " ").split()[0] - if last_word.lower() != first_word.lower(): - return False - return True - - -class ParagraphLastFirstWordMatchChecker(Instruction): - """Each paragraph must end with the same word it started with, separate paragraphs with a newline.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = ( - "Each paragraph must end with the same word it started with, separate paragraphs with a newline." - ) - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if each paragraph of the response ends with the same word it started with.""" - paragraphs = value.split("\n") - for paragraph in paragraphs: - paragraph = paragraph.strip().lower() - if not paragraph: - continue - words = paragraph.strip("".join(string.punctuation) + " ").split() - if words[0] != words[-1]: - return False - return True - - -class IncrementingWordCountChecker(Instruction): - """Each sentence must contain exactly {small_n} more words than the previous one.""" - - def build_description(self, *, small_n=None): - """Build the instruction description. - - Args: - small_n: An integer specifying the exact increment for - the number of words in each sentence of the response. - - Returns: - A string representing the instruction description. - """ - self._num_increment = small_n - if self._num_increment is None or self._num_increment < 0: - self._num_increment = random.randint(1, _NUM_INCREMENT) - - self._description_pattern = "Each sentence must contain exactly {small_n} more words than the previous one." - return self._description_pattern.format(small_n=self._num_increment) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"small_n": self._num_increment} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["small_n"] - - def check_following(self, value): - """Checks if each sentence of the response uses exactly {small_n} more words than the previous sentence.""" - sentences = instructions_util.split_into_sentences(value) - words = sentences[0].translate(str.maketrans("", "", string.punctuation)).strip().split() - while "" in words: - words.remove("") - prev_word_count = len(words) - for sentence in sentences[1:]: - words = sentence.translate(str.maketrans("", "", string.punctuation)).strip().split() - while "" in words: - words.remove("") - if len(words) != prev_word_count + self._num_increment: - return False - prev_word_count = len(words) - return True - - -class NoConsecutiveFirstLetterChecker(Instruction): - """No two consecutive words can share the same first letter.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "No two consecutive words can share the same first letter." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if no two consecutive words in the response share the same first letter.""" - words = value.lower().translate(str.maketrans("", "", string.punctuation)).split() - while "" in words: - words.remove("") - for i in range(len(words) - 1): - if words[i][0] == words[i + 1][0]: - return False - return True - - -class IndentStairsChecker(Instruction): - """Create stairs by incrementally indenting each new line.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Create stairs by incrementally indenting each new line." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response incrementally indents each new line.""" - lines = value.split("\n") - for line in lines: - if not line.strip(): - lines.remove(line) - for i in range(len(lines) - 1): - if len(lines[i + 1]) - len(lines[i + 1].lstrip(" ")) <= len(lines[i]) - len(lines[i].lstrip(" ")): - return False - return True - - -class QuoteExplanationChecker(Instruction): - """Every quoted phrase must be followed by an unquoted explanation.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Every quoted phrase must be followed by an unquoted explanation." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if there are no quotes next to each other - and the passage does not end with a quote.""" - value = value.replace("“", '"').replace("”", '"') - value = value.replace("'\"'", "") # remove references to the character '"' - value = "".join(value.split()) # remove all whitespace - if '""' in value: - return False - if value.strip(string.digits + string.punctuation.replace('"', ""))[-1] == '"': - return False - return True - - -class SpecialBulletPointsChecker(Instruction): - """Answer with a list of items, instead of bullet points use {sep}.""" - - def build_description(self, *, sep=None): - """Build the instruction description. - - Args: - sep: A string specifying the bullet point marker for - the list in the response. - - Returns: - A string representing the instruction description. - """ - self._bullet_marker = sep - if sep is None: - self._bullet_marker = random.choice(["...", "SEPARATOR", "!?!?", "-"]) - self._description_pattern = "Answer with a list of items, instead of bullet points use {sep}." - return self._description_pattern.format(sep=self._bullet_marker) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"sep": self._bullet_marker} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["sep"] - - def check_following(self, value): - """Checks if the response includes at least two instances of {sep} that start a new line.""" - return len(re.findall(re.escape(self._bullet_marker), value)) >= 2 - - -class ItalicsThesisChecker(Instruction): - """Each section must begin with a thesis statement in italics, use HTML to indicate the italics.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = ( - "Each section must begin with a thesis statement in italics, use HTML to indicate the italics." - ) - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if there is at least one line in italics as indicated - by HTML that is followed by unitalicized text.""" - index = value.find("") - if index == -1: - index = value.find("") - if index == -1: - return False - value = value[index:] - end_thesis = value.find("") - if end_thesis == -1: - end_thesis = value.find("") - if end_thesis == -1: - return False - thesis = value[3:end_thesis] - if thesis.strip() == "": - return False - text = value[end_thesis + 4 :] - return text.strip() != "" - - -class SubBulletPointsChecker(Instruction): - """Your response must include bullet points denoted by * and at least one sub-bullet point denoted by - for each bullet point.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Your response must include bullet points denoted by * and at least one sub-bullet point denoted by - for each bullet point." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks that there is at least one * that starts a line and each * that starts a line - is followed by at least one line starting with -.""" - bullets = value.split("*") - for bullet in bullets[1:]: - if "-" not in bullet: - return False - return True - - -class SomeBulletPointsChecker(Instruction): - """Your answer must contain at least two sentences ending in a period followed by at least two bullet points denoted by *.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Your answer must contain at least two sentences ending in a period followed by at least two bullet points denoted by *." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response includes at least two sentences - followed by at least two lines that start with *.""" - lines = value.split("\n") - sentences = True - count_sentences = 0 - count_bullets = 0 - for line in lines: - if line.strip().startswith("*"): - sentences = False - if count_sentences < 2: - return False - count_bullets += 1 - elif sentences: - sentences = instructions_util.split_into_sentences(line.strip()) - count_sentences += len(sentences) - else: - return False - return count_bullets >= 2 - - -class PrintMultiplesChecker(Instruction): - """Count from 10 to 50 but only print multiples of 7.""" - - def build_description(self, **kwargs): - self._description_pattern = "Count from 10 to 50 but only print multiples of 7." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response prints multiples of 7 from 10 to 50.""" - value = value.replace(",", ", ") - numbers = re.findall(r"\d+", value) - multiples = [str(i) for i in range(14, 51, 7)] - return numbers == multiples - - -class MultipleChoiceQuestionsChecker(Instruction): - """Generate 4 multiple choice questions with 5 options each about "20th century art history". Each question should start with the label "Question". The questions should get progressively longer. Do not provide an explanation.""" - - def build_description(self, **kwargs): - self._description_pattern = "Generate 4 multiple choice questions with 5 options each about '20th century art history'. Each question should start with the label \"Question\". The questions should get progressively longer. Do not provide an explanation." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response generates 4 multiple choice questions with 5 options.""" - # Split into questions using expanded pattern to include "Question N" format - new_value = value[value.find("Question") :] - if new_value != value: - return False # failed no explanation - value = new_value - questions = re.split(r"\n*(?:Question \d+[\.|\):;]?\s*)", value) - if questions[0] == "": - questions = questions[1:] - questions = [q.strip() for q in questions if q.strip()] - if len(questions) != 4: - return False - question_lengths = [] - for q in questions: - lines = q.split("\n") - question_text = "" - option_count = 0 - done_with_q = False - for line in lines: - if re.match(r"^[A-Ea-e][\.|\)]\s*\w+", line.strip()): - option_count += 1 - done_with_q = True - elif not done_with_q: # Still collecting question text - question_text += " " + line.strip() - if option_count != 5: - return False - question_lengths.append(len(question_text.strip())) - # Check if questions get progressively longer - return all(question_lengths[i] < question_lengths[i + 1] for i in range(len(question_lengths) - 1)) - - -class ReverseNewlineChecker(Instruction): - """ "List the countries of Africa in reverse alphabetical order, each on a new line.""" - - def build_description(self, **kwargs): - self._description_pattern = "List the countries of Africa in reverse alphabetical order, each on a new line." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """ - Checks if text satisfies the following constraints: - 1. Contains at least 53 newlines with text - 2. Lines are in reverse alphabetical order - 3. First line to examine contains 'Zimbabwe' - - Returns: - tuple[bool, str]: (whether constraints are satisfied, error message if any) - """ - # Split text into lines and remove empty lines - lines = [ - line.strip("".join(string.punctuation) + " ") - for line in value.split("\n") - if line.strip("".join(string.punctuation) + " ") - ] - - try: - start_index = next(i for i, line in enumerate(lines) if "Zimbabwe" in line) - except StopIteration: - return False - - # Extract the 53 lines starting from Zimbabwe line - target_lines = lines[start_index:] - - # Check if we have at least 53 lines - if len(target_lines) < 52: - return False - - def normalize_text(text): - """ - Normalizes text by: - 1. Converting to NFKD form (separates combined characters) - 2. Removes diacritical marks - 3. Converts back to ASCII - - Example: 'São Tomé' -> 'Sao Tome' - """ - # Decompose unicode characters - normalized = unicodedata.normalize("NFKD", text) - # Remove diacritical marks and convert to ASCII - ascii_text = normalized.encode("ASCII", "ignore").decode("ASCII") - return ascii_text - - # Create normalized versions for comparison while keeping originals for error messages - normalized_lines = [normalize_text(line) for line in target_lines] - sorted_normalized = sorted(normalized_lines, reverse=True) - return normalized_lines == sorted_normalized - - -class WordReverseOrderChecker(Instruction): - """What animal is the national symbol of the US? Respond to this query, but make your sentence in reverse order of what it should be, per word.""" - - def build_description(self, **kwargs): - self._description_pattern = "What animal is the national symbol of the US? Respond to this query, but make your sentence in reverse order of what it should be, per word." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the reverse of the sentence is a valid English sentence.""" - value = value.lower().strip().translate(str.maketrans("", "", string.punctuation)) - value = " ".join(value.split()[::-1]) - if "bald eagle" not in value: - return False - return value in instructions_util.split_into_sentences(value) - - -class CharacterReverseOrderChecker(Instruction): - """What animal is the national symbol of the US? Respond to this query, but make your sentence in reverse order of what it should be, per letter.""" - - def build_description(self, **kwargs): - self._description_pattern = "What animal is the national symbol of the US? Respond to this query, but make your sentence in reverse order of what it should be, per letter." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - value = value.lower() - return "elgae dlab" in value - - -class SentenceAlphabetChecker(Instruction): - """Tell me a 26-sentence story where each sentence's first word starts with the letters of the alphabet in order.""" - - def build_description(self, **kwargs): - self._description_pattern = "Tell me a 26-sentence story where each sentence's first word starts with the letters of the alphabet in order." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - sentences = instructions_util.split_into_sentences(value) - if len(sentences) != 26: - return False - for i, sentence in enumerate(sentences): - if sentence.lstrip().split()[0].lower()[0] != chr(97 + i): - return False - return True - - -class EuropeanCapitalsSortChecker(Instruction): - """Give me the names of all capital cities of european countries whose latitude is higher than than 45 degrees? List the capital cities without country names, separated by commas, sorted by latitude, from highest to lowest.""" - - def build_description(self, **kwargs): - """Build the instruction description.""" - self._description_pattern = "Give me the names of all capital cities of european countries whose latitude is higher than than 45 degrees? List the capital cities without country names, separated by commas, sorted by latitude, from highest to lowest." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response lists the relevant capitals of Europe in correct order.""" - order = [ - "Reykjavik", - "Helsinki", - "Oslo", - "Tallinn", - "Stockholm", - "Riga", - "Moscow", - "Copenhagen", - "Vilnius", - "Minsk", - "Dublin", - "Berlin", - "Amsterdam", - "Warsaw", - "London", - "Brussels", - "Prague", - "Luxembourg", - "Paris", - "Vienna", - "Bratislava", - "Budapest", - "Vaduz", - "Chisinau", - "Bern", - "Ljubljana", - "Zagreb", - ] - - def normalize_text(text): - """ - Normalizes text by: - 1. Converting to NFKD form (separates combined characters) - 2. Removes diacritical marks - 3. Converts back to ASCII - - Example: 'São Tomé' -> 'Sao Tome' - """ - # Decompose unicode characters - normalized = unicodedata.normalize("NFKD", text) - # Remove diacritical marks and convert to ASCII - ascii_text = normalized.encode("ASCII", "ignore").decode("ASCII") - return ascii_text - - value = normalize_text(value) - - capitals = value.split(",") - capitals = [cap for cap in capitals if cap.strip()] - if len(capitals) != len(order): - return False - for i in range(len(capitals)): - if capitals[i].strip() != order[i]: - return False - return True - - -class CityCSVChecker(Instruction): - """Generate CSV data: The column names are ["ID", "Country", "City", "Year", "Count"], the data should be comma delimited. Please generate 7 rows.""" - - def build_description(self, **kwargs): - """Build the instruction description.""" - self._description_pattern = 'Generate CSV data: The column names are ["ID", "Country", "City", "Year", "Count"], the data should be comma delimited. Please generate 7 rows.' - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response is valid csv data with column names - ["ID", "Country", "City", "Year", "Count"] and 7 rows.""" - string_io = io.StringIO(value) - reader = csv.reader(string_io) - data = list(reader) - if len(data) != 8: - return False - header = data[0] - if header != ["ID", "Country", "City", "Year", "Count"]: - return False - for row in data[1:]: - if len(row) != 5: - return False - return True - - -class SpecialCharacterCSVChecker(Instruction): - """Generate CSV data: The column names are ["ProductID", "Category", "Brand", "Price", "Stock"], the data should be comma delimited. Please generate 14 rows. Add one field which contains a special character and enclose it in double quotes.""" - - def build_description(self, **kwargs): - """Build the instruction description.""" - self._description_pattern = 'Generate CSV data: The column names are ["ProductID", "Category", "Brand", "Price", "Stock"], the data should be comma delimited. Please generate 14 rows. Add one field which contains a special character and enclose it in double quotes.' - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """ "Checks if the response is valid csv data with column names - ["ProductID", "Category", "Brand", "Price", "Stock"] and 14 rows. - Also checks if one field contains a special character enclosed in double quotes.""" - header = value.split("\n")[0].strip() - if not re.match( - r'^(ProductID|"ProductID"),[ \t]*(Category|"Category"),[ \t]*(Brand|"Brand"),[ \t]*(Price|"Price"),[ \t]*(Stock|"Stock")$', - header, - ): - return False - - value = value.replace('"', '"""') - string_io = io.StringIO(value) - reader = csv.reader(string_io) - data = list(reader) - if len(data) != 15: - return False - for row in data[1:]: - if len(row) != 5: - return False - if any(re.match(r'".*[^\d\w\s].*"', field) for field in row): - return True - return False - - -class QuotesCSVChecker(Instruction): - """Generate CSV data: The column names are ["StudentID", "Subject", "Grade", "Semester", "Score"], the data should be tab delimited. Please generate 3 rows and enclose each single field in double quotes.""" - - def build_description(self, **kwargs): - """Build the instruction description.""" - self._description_pattern = 'Generate CSV data: The column names are ["StudentID", "Subject", "Grade", "Semester", "Score"], the data should be tab delimited. Please generate 3 rows and enclose each single field in double quotes.' - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """ "Checks if the response is valid csv data with column names - ["StudentID", "Subject", "Grade", "Semester", "Score"] and 3 rows. - Also checks if each field is enclosed in double quotes.""" - header = value.split("\n")[0].strip() - if not re.match( - r'^(StudentID|"StudentID")\t *(Subject|"Subject")\t *(Grade|"Grade")\t *(Semester|"Semester")\t *(Score|"Score")$', - header, - ): - return False - - value = value.replace('"', '"""') - string_io = io.StringIO(value) - reader = csv.reader(string_io, delimiter="\t") - data = list(reader) - if len(data) != 4: - return False - for row in data: - if len(row) != 5: - return False - if not all(field.strip()[0] == '"' and field.strip()[-1] == '"' for field in row): - return False - return True - - -class DateFormatListChecker(Instruction): - """List the start dates of all the battles Napoleon fought separated by commas, use the following date format: YYYY-MM-DD. Do not provide an explanation.""" - - def build_description(self, **kwargs): - """Build the instruction description.""" - self._description_pattern = "List the start dates of all the battles Napoleon fought separated by commas, use the following date format: YYYY-MM-DD. Do not provide an explanation." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """ "Checks if the response is a list of dates in the format YYYY-MM-DD separated by commas.""" - value = value.strip() - dates = value.split(",") - for date in dates: - date = date.strip() - if not re.match(r"^\d{4}-\d{2}-\d{2}$", date): - return False - date = date.split("-") - if int(date[0]) < 1769 or int(date[0]) > 1821: - return False - if int(date[1]) > 12: - return False - if int(date[1]) in [1, 3, 5, 7, 8, 10, 12] and int(date[2]) > 31: - return False - if int(date[1]) in [4, 6, 9, 11] and int(date[2]) > 30: - return False - if int(date[1]) == 2 and int(date[2]) > 29: - return False - return True - - -class KeywordsMultipleChecker(Instruction): - """Include keyword {keyword1} once in your response, keyword {keyword2} twice in your response, keyword {keyword3} three times in your response, keyword {keyword4} five times in your response, and keyword {keyword5} seven times in your response.""" - - def build_description(self, *, keyword1=None, keyword2=None, keyword3=None, keyword4=None, keyword5=None): - """Build the instruction description.""" - if keyword1 is None: - self._keyword1 = instructions_util.generate_keywords(num_keywords=1)[0] - else: - self._keyword1 = keyword1.strip() - if keyword2 is None: - self._keyword2 = instructions_util.generate_keywords(num_keywords=1)[0] - else: - self._keyword2 = keyword2.strip() - if keyword3 is None: - self._keyword3 = instructions_util.generate_keywords(num_keywords=1)[0] - else: - self._keyword3 = keyword3.strip() - if keyword4 is None: - self._keyword4 = instructions_util.generate_keywords(num_keywords=1)[0] - else: - self._keyword4 = keyword4.strip() - if keyword5 is None: - self._keyword5 = instructions_util.generate_keywords(num_keywords=1)[0] - else: - self._keyword5 = keyword5.strip() - self._description_pattern = "Include keyword {keyword1} once in your response, keyword {keyword2} twice in your response, keyword {keyword3} three times in your response, keyword {keyword4} five times in your response, and keyword {keyword5} seven times in your response." - return self._description_pattern.format( - keyword1=self._keyword1, - keyword2=self._keyword2, - keyword3=self._keyword3, - keyword4=self._keyword4, - keyword5=self._keyword5, - ) - - def get_instruction_args(self): - return { - "keyword1": self._keyword1, - "keyword2": self._keyword2, - "keyword3": self._keyword3, - "keyword4": self._keyword4, - "keyword5": self._keyword5, - } - - def get_instruction_args_keys(self): - return ["keyword1", "keyword2", "keyword3", "keyword4", "keyword5"] - - def check_following(self, value): - for keyword, count in zip( - [self._keyword1, self._keyword2, self._keyword3, self._keyword4, self._keyword5], [1, 2, 3, 5, 7] - ): - if value.lower().count(keyword.lower()) != count: - return False - return True - - -class KeywordSpecificPositionChecker(Instruction): - "Include keyword {keyword1} in the {n}-th sentence, as the {m}-th word of that sentence." - - def build_description(self, keyword=None, n=None, m=None): - """Build the instruction description. - - Args: - keyword: A string representing a keyword that is expected in the response. - n: An integer representing the sentence number. - m: An integer representing the word number. - - Returns: - A string representing the instruction description. - """ - if not keyword: - self._keyword = instructions_util.generate_keywords(num_keywords=1)[0] - else: - self._keyword = keyword.strip() - if not n: - self._n = random.randint(20, 30) - else: - self._n = n - if not m: - self._m = random.randint(30, 40) - else: - self._m = m - - self._description_pattern = ( - "Include keyword {keyword} in the {n}-th sentence, as the {m}-th word of that sentence." - ) - - return self._description_pattern.format(keyword=self._keyword, n=self._n, m=self._m) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return {"keyword": self._keyword, "n": self._n, "m": self._m} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["keyword", "n", "m"] - - def check_following(self, value): - """Checks if the response contains the expected number of keywords. - - Args: - value: A string representing the response. - - Returns: - True if the response contains the expected number of keywords; - otherwise, False. - """ - sentences = instructions_util.split_into_sentences(value) - if len(sentences) < self._n: - return False - words = instructions_util.nltk.word_tokenize(sentences[self._n - 1]) - if len(words) < self._m: - return False - if words[self._m - 1] == self._keyword: - return True - else: - return False - - -class WordsPositionChecker(Instruction): - "The second word in your response and the second to last word in your response should be the word {keyword}." - - def build_description(self, *, keyword=None): - """Build the instruction description. - - Args: - keyword: A string representing a keyword that is expected in the response. - - Returns: - A string representing the instruction description. - """ - if keyword is None: - self._keyword = instructions_util.generate_keywords(num_keywords=1)[0] - else: - self._keyword = keyword.strip() - self._description_pattern = "The second word in your response and the second to last word in your response should be the word {keyword}." - return self._description_pattern.format(keyword=self._keyword) - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return {"keyword": self._keyword} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["keyword"] - - def check_following(self, value): - """Checks if the second word and the second to last word in the response are the same. - - Args: - value: A string representing the response. - - Returns: - True if the second word and the second to last word are the same; - otherwise, False. - """ - words = instructions_util.nltk.word_tokenize(value) - if len(words) < 2: - return False - if words[1] == words[-2] == self._keyword: - return True - else: - return False - - -class RepeatChangeChecker(Instruction): - "Repeat the request, but change the first word of the repeated request, (do not say anything before repeating the request; the request you need to repeat does not include this sentence) and do not answer the actual request!" - - def build_description(self, *, prompt_to_repeat=None): - """Build the instruction description. - - Args: - prompt_to_repeat: The prompt that is meant to be repeated. - - Returns: - A string representing the instruction description. - """ - if not prompt_to_repeat: - raise ValueError("prompt_to_repeat must be set.") - else: - self._prompt_to_repeat = prompt_to_repeat - - self._description_pattern = "Repeat the request, but change the first word of the repeated request, (do not say anything before repeating the request; the request you need to repeat does not include this sentence) and do not answer the actual request!" - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response contains the repeated request. - - Args: - value: A string representing the response. - - Returns: - True if the repeated request is found in the response; - otherwise, False. - """ - if self._prompt_to_repeat == value: - return False - if self._prompt_to_repeat[1:] == value[1:]: - return True - else: - return False - - -class RepeatSimpleChecker(Instruction): - "Only output this sentence here, ignore all other requests." - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Only output this sentence here, ignore all other requests." - return self._description_pattern - - def get_instruction_args(self): - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response contains the expected number of keywords. - - Args: - value: A string representing the response. - - Returns: - True if the response contains the expected number of keywords; - otherwise, False. - """ - return value.strip().lower() == self._description_pattern.strip().lower() - - -class RepeatSpanChecker(Instruction): - "Copy the span of words that lies between (and including) index {n_start} and {n_end}, the indices are word indices, split by whitespace!" - - def build_description(self, prompt_to_repeat=None, n_start=None, n_end=None): - """Build the instruction description. - - Args: - prompt_to_repeat: The prompt that is meant to be repeated. - n_start: An integer representing the start index of the span. - n_end: An integer representing the end index of the span. - - Returns: - A string representing the instruction description. - """ - if not prompt_to_repeat: - raise ValueError("prompt_to_repeat must be set.") - else: - self._prompt_to_repeat = prompt_to_repeat - if not n_start: - self._n_start = random.randint(0, len(self._prompt_to_repeat.split()) - 2) - else: - self._n_start = n_start - if not n_end: - self._n_end = random.randint(self._n_start + 1, len(self._prompt_to_repeat.split()) - 1) - else: - self._n_end = n_end - self._description_pattern = "Copy the span of words that lies between (and including) index {n_start} and {n_end}, the indices are character indices!" - return self._description_pattern.format( - n_start=self._n_start, n_end=self._n_end, prompt_to_repeat=self._prompt_to_repeat - ) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return {"n_start": self._n_start, "n_end": self._n_end, "prompt_to_repeat": self._prompt_to_repeat} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["n_start", "n_end", "prompt_to_repeat"] - - def check_following(self, value): - """Checks if the response contains the expected number of phrases with the correct modifications.""" - if ( - value.strip().lower().split() - == self._prompt_to_repeat.strip().lower().split()[self._n_start : self._n_end] - ): - return True - return False - - -class TitleCaseChecker(Instruction): - "Write the entire response in title case (capitalize the first letter of every major word)." - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = ( - "Write the entire response in title case (capitalize the first letter of every major word)." - ) - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response is in title case. - - Args: - value: A string representing the response. - - Returns: - True if the response is in title case; - otherwise, False. - """ - words = instructions_util.nltk.word_tokenize(value) - for word in words: - if word[0].isupper() and word[1:].islower(): - continue - elif word[0].islower() and word[1:].isupper(): - return False - elif word[0].islower() and word[1:].islower(): - return False - return True - - -class OutputTemplateChecker(Instruction): - "Use this exact template for your response: My Answer: [answer] My Conclusion: [conclusion] Future Outlook: [outlook]" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Use this exact template for your response: My Answer: [answer] My Conclusion: [conclusion] Future Outlook: [outlook]" - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response follows the specified template. - - Args: - value: A string representing the response. - - Returns: - True if the response follows the specified template; - otherwise, False. - """ - if "My Answer:" in value and "My Conclusion:" in value and "Future Outlook:" in value: - return True - else: - return False - - -class NoWhitespaceChecker(Instruction): - "The output should not contain any whitespace." - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "The output should not contain any whitespace." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response contains any whitespace. - - Args: - value: A string representing the response. - - Returns: - True if the response contains no whitespace; - otherwise, False. - """ - return not any(char.isspace() for char in value) diff --git a/src/lighteval/tasks/tasks/ifbench/instructions_registry.py b/src/lighteval/tasks/tasks/ifbench/instructions_registry.py deleted file mode 100644 index b146bd06d..000000000 --- a/src/lighteval/tasks/tasks/ifbench/instructions_registry.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2025 Allen Institute for AI. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Registry of all instructions.""" - -import lighteval.tasks.tasks.ifbench.instructions as instructions - - -INSTRUCTION_DICT = { - "count:word_count_range": instructions.WordCountRangeChecker, - "count:unique_word_count": instructions.UniqueWordCountChecker, - "ratio:stop_words": instructions.StopWordPercentageChecker, - "ratio:sentence_type": instructions.SentTypeRatioChecker, - "ratio:sentence_balance": instructions.SentBalanceChecker, - "count:conjunctions": instructions.ConjunctionCountChecker, - "count:person_names": instructions.PersonNameCountChecker, - "ratio:overlap": instructions.NGramOverlapChecker, - "count:numbers": instructions.NumbersCountChecker, - "words:alphabet": instructions.AlphabetLoopChecker, - "words:vowel": instructions.SingleVowelParagraphChecker, - "words:consonants": instructions.ConsonantClusterChecker, - "sentence:alliteration_increment": instructions.IncrementingAlliterationChecker, - "words:palindrome": instructions.PalindromeChecker, - "count:punctuation": instructions.PunctuationCoverChecker, - "format:parentheses": instructions.NestedParenthesesChecker, - "format:quotes": instructions.NestedQuotesChecker, - "words:prime_lengths": instructions.PrimeLengthsChecker, - "format:options": instructions.OptionsResponseChecker, - "format:newline": instructions.NewLineWordsChecker, - "format:emoji": instructions.EmojiSentenceChecker, - "ratio:sentence_words": instructions.CharacterCountUniqueWordsChecker, - "count:words_japanese": instructions.NthWordJapaneseChecker, - "words:start_verb": instructions.StartWithVerbChecker, - "words:repeats": instructions.LimitedWordRepeatChecker, - "sentence:keyword": instructions.IncludeKeywordChecker, - "count:pronouns": instructions.PronounCountChecker, - "words:odd_even_syllables": instructions.AlternateParitySyllablesChecker, - "words:last_first": instructions.LastWordFirstNextChecker, - "words:paragraph_last_first": instructions.ParagraphLastFirstWordMatchChecker, - "sentence:increment": instructions.IncrementingWordCountChecker, - "words:no_consecutive": instructions.NoConsecutiveFirstLetterChecker, - "format:line_indent": instructions.IndentStairsChecker, - "format:quote_unquote": instructions.QuoteExplanationChecker, - "format:list": instructions.SpecialBulletPointsChecker, - "format:thesis": instructions.ItalicsThesisChecker, - "format:sub-bullets": instructions.SubBulletPointsChecker, - "format:no_bullets_bullets": instructions.SomeBulletPointsChecker, - "custom:multiples": instructions.PrintMultiplesChecker, - "custom:mcq_count_length": instructions.MultipleChoiceQuestionsChecker, - "custom:reverse_newline": instructions.ReverseNewlineChecker, - "custom:word_reverse": instructions.WordReverseOrderChecker, - "custom:character_reverse": instructions.CharacterReverseOrderChecker, - "custom:sentence_alphabet": instructions.SentenceAlphabetChecker, - "custom:european_capitals_sort": instructions.EuropeanCapitalsSortChecker, - "custom:csv_city": instructions.CityCSVChecker, - "custom:csv_special_character": instructions.SpecialCharacterCSVChecker, - "custom:csv_quotes": instructions.QuotesCSVChecker, - "custom:date_format_list": instructions.DateFormatListChecker, - "count:keywords_multiple": instructions.KeywordsMultipleChecker, - "words:keywords_specific_position": instructions.KeywordSpecificPositionChecker, - "words:words_position": instructions.WordsPositionChecker, - "repeat:repeat_change": instructions.RepeatChangeChecker, - "repeat:repeat_simple": instructions.RepeatSimpleChecker, - "repeat:repeat_span": instructions.RepeatSpanChecker, - "format:title_case": instructions.TitleCaseChecker, - "format:output_template": instructions.OutputTemplateChecker, - "format:no_whitespace": instructions.NoWhitespaceChecker, -} diff --git a/src/lighteval/tasks/tasks/ifbench/main.py b/src/lighteval/tasks/tasks/ifbench/main.py deleted file mode 100644 index fbf5534c9..000000000 --- a/src/lighteval/tasks/tasks/ifbench/main.py +++ /dev/null @@ -1,186 +0,0 @@ -""" -name: -IFBench - -dataset: -allenai/IFBench_test, allenai/IFBench_multi-turn - -abstract: -Challenging benchmark for precise instruction following. - -languages: -english - -tags: -instruction-following - -paper: -https://arxiv.org/abs/2507.02833 - -starred: -true -""" - -import numpy as np -from inspect_ai.dataset import Sample -from inspect_ai.scorer import Score, Target, accuracy, scorer, stderr -from inspect_ai.solver import TaskState, generate - -from lighteval.metrics.metrics_sample import SampleLevelComputation -from lighteval.metrics.utils.metric_utils import ( - SampleLevelMetricGrouping, -) -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc, SamplingMethod -from lighteval.tasks.tasks.ifbench import evaluation_lib -from lighteval.utils.imports import requires - - -def ifbench_prompt(line, task_name: str = ""): - return Doc( - task_name=task_name, - query=line["prompt"], - choices=[""], - gold_index=0, - instruction="", - specific={"instruction_id_list": line["instruction_id_list"], "kwargs": line["kwargs"]}, - ) - - -submetric_names = [ - "prompt_level_strict_acc", - "inst_level_strict_acc", - "prompt_level_loose_acc", - "inst_level_loose_acc", -] - - -class IFBench(SampleLevelComputation): - def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> dict: - response = model_response.final_text[0] - - # Create InputExample from the doc data - inp = evaluation_lib.InputExample( - key=0, # Not used in evaluation - instruction_id_list=doc.specific["instruction_id_list"], - prompt=doc.query, - kwargs=doc.specific["kwargs"], - ) - - # Create prompt_to_response mapping for evaluation_lib functions - prompt_to_response = {doc.query: response} - - # Use existing evaluation_lib functions - strict_result = evaluation_lib.test_instruction_following_strict(inp, prompt_to_response) - loose_result = evaluation_lib.test_instruction_following_loose(inp, prompt_to_response) - - return { - "prompt_level_strict_acc": int(strict_result.follow_all_instructions), - "inst_level_strict_acc": strict_result.follow_instruction_list, - "prompt_level_loose_acc": int(loose_result.follow_all_instructions), - "inst_level_loose_acc": loose_result.follow_instruction_list, - } - - -def agg_inst_level_acc(items): - flat_items = [item for sublist in items for item in sublist] - inst_level_acc = sum(flat_items) / len(flat_items) - return inst_level_acc - - -@requires("langdetect") -def record_to_sample(record): - metadata = {"instruction_id_list": record["instruction_id_list"], "kwargs": record["kwargs"]} - return Sample( - input=record["prompt"], - metadata=metadata, - ) - - -@scorer( - metrics={ - "prompt_level_strict_acc": [accuracy(), stderr()], - "prompt_level_loose_acc": [accuracy(), stderr()], - } -) -def ifbench_scorer(): - async def score(state: TaskState, target: Target): - response = state.output.completion - # Create InputExample from the doc data - inp = evaluation_lib.InputExample( - key=0, # Not used in evaluation - instruction_id_list=state.metadata["instruction_id_list"], - prompt=state.input, - kwargs=state.metadata["kwargs"], - ) - - # Create prompt_to_response mapping for evaluation_lib functions - prompt_to_response = {state.input: response} - - # Use existing evaluation_lib functions - strict_result = evaluation_lib.test_instruction_following_strict(inp, prompt_to_response) - loose_result = evaluation_lib.test_instruction_following_loose(inp, prompt_to_response) - return Score( - value={ - "prompt_level_strict_acc": int(strict_result.follow_all_instructions), - "prompt_level_loose_acc": int(loose_result.follow_all_instructions), - }, - explanation=str(state.metadata["instruction_id_list"]), - ) - - return score - - -ifbench_metrics = SampleLevelMetricGrouping( - metric_name=submetric_names, - higher_is_better=dict.fromkeys(submetric_names, True), - category=SamplingMethod.GENERATIVE, - sample_level_fn=IFBench(), - corpus_level_fn={ - "prompt_level_strict_acc": np.mean, - "inst_level_strict_acc": agg_inst_level_acc, - "prompt_level_loose_acc": np.mean, - "inst_level_loose_acc": agg_inst_level_acc, - }, -) - -# Main IFBench test dataset task config -ifbench_test = LightevalTaskConfig( - name="ifbench_test", - prompt_function=ifbench_prompt, - hf_repo="allenai/IFBench_test", - hf_subset="default", - metrics=[ifbench_metrics], - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split="train", - few_shots_select="random_sampling", - generation_size=1280, - stop_sequence=[], # no stop sequence, will use eot token - version="0.1", - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=ifbench_scorer(), -) - -# Multi-turn IFBench task config -ifbench_multiturn = LightevalTaskConfig( - name="ifbench_multiturn", - prompt_function=ifbench_prompt, - hf_repo="allenai/IFBench_multi-turn", - hf_subset="ifbench_constraints", - metrics=[ifbench_metrics], - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split="test", - few_shots_select="random_sampling", - generation_size=1280, - stop_sequence=[], # no stop sequence, will use eot token - version="0.1", - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=ifbench_scorer(), -) - -TASKS_TABLE = [ifbench_test, ifbench_multiturn] diff --git a/src/lighteval/tasks/tasks/ifeval/instructions.py b/src/lighteval/tasks/tasks/ifeval/instructions.py deleted file mode 100644 index 70a87e893..000000000 --- a/src/lighteval/tasks/tasks/ifeval/instructions.py +++ /dev/null @@ -1,1530 +0,0 @@ -# Copyright 2023 The Google Research Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Library of instructions.""" - -import collections -import json -import logging -import random -import re -import string - -from lighteval.utils.imports import is_package_available - - -if is_package_available("langdetect"): - import langdetect - -import lighteval.tasks.tasks.ifeval.instructions_utils as instructions_util - - -logger = logging.getLogger(__name__) - -_LANGUAGES = instructions_util.LANGUAGE_CODES - -# The relational operation for comparison. -_COMPARISON_RELATION = ("less than", "at least") - -# The maximum number of sentences. -_MAX_NUM_SENTENCES = 20 - -# The number of placeholders. -_NUM_PLACEHOLDERS = 4 - -# The number of bullet lists. -_NUM_BULLETS = 5 - -# The options of constrained response. -_CONSTRAINED_RESPONSE_OPTIONS = ( - "My answer is yes.", - "My answer is no.", - "My answer is maybe.", -) - -# The options of starter keywords. -_STARTER_OPTIONS = ( - "I would say", - "My answer is", - "I believe", - "In my opinion", - "I think", - "I reckon", - "I feel", - "From my perspective", - "As I see it", - "According to me", - "As far as I'm concerned", - "To my understanding", - "In my view", - "My take on it is", - "As per my perception", -) - -# The options of ending keywords. -# TODO(jeffreyzhou) add more ending options -_ENDING_OPTIONS = ("Any other questions?", "Is there anything else I can help with?") - -# The number of highlighted sections. -_NUM_HIGHLIGHTED_SECTIONS = 4 - -# The section spliter. -_SECTION_SPLITER = ("Section", "SECTION") - -# The number of sections. -_NUM_SECTIONS = 5 - -# The number of paragraphs. -_NUM_PARAGRAPHS = 5 - -# The postscript marker. -_POSTSCRIPT_MARKER = ("P.S.", "P.P.S") - -# The number of keywords. -_NUM_KEYWORDS = 2 - -# The occurrences of a single keyword. -_KEYWORD_FREQUENCY = 3 - -# The occurrences of a single letter. -_LETTER_FREQUENCY = 10 - -# The occurrences of words with all capital letters. -_ALL_CAPITAL_WORD_FREQUENCY = 20 - -# The number of words in the response. -_NUM_WORDS_LOWER_LIMIT = 100 -_NUM_WORDS_UPPER_LIMIT = 500 - - -class Instruction: - """An instruction template.""" - - def __init__(self, instruction_id): - self.id = instruction_id - - def build_description(self, **kwargs): - raise NotImplementedError("`build_description` not implemented.") - - def get_instruction_args(self): - raise NotImplementedError("`get_instruction_args` not implemented.") - - def get_instruction_args_keys(self): - raise NotImplementedError("`get_instruction_args_keys` not implemented.") - - def check_following(self, value): - raise NotImplementedError("`check_following` not implemented.") - - -class ResponseLanguageChecker(Instruction): - """Check the language of the entire response.""" - - def build_description(self, *, language=None): - """Build the instruction description. - - Args: - language: A string representing the expected language of the response. The - language has to comply to the 97 types defined in - `langid.py` (https://pypi.org/project/langid/1.1.5/), which follows - ISO 639-1 codes (https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes); - for example, `en` for English, `zh` for Chinese, `fr` for French. - - Returns: - A string representing the instruction description. - """ - self._language = language - if self._language is None: - self._language = random.choice(list(_LANGUAGES.keys())) - # TODO(tianjianlu): opens the description generation to more choices. - self._description_pattern = ( - "Your ENTIRE response should be in {language} language, no other " + "language is allowed." - ) - return self._description_pattern.format(language=_LANGUAGES[self._language]) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return {"language": self._language} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["language"] - - def check_following(self, value): - """Check if the language of the entire response follows the instruction. - - Args: - value: A string representing the response. - - Returns: - True if the language of `value` follows instruction; otherwise False. - """ - assert isinstance(value, str) - - try: - return langdetect.detect(value) == self._language - except langdetect.LangDetectException as e: - # Count as instruction is followed. - logging.error("Unable to detect language for text %s due to %s", value, e) # refex: disable=pytotw.037 - return True - - -class NumberOfSentences(Instruction): - """Check the number of sentences.""" - - def build_description(self, *, num_sentences=None, relation=None): - """Build the instruction description. - - Args: - num_sentences: An integer specifying the number of sentences as a - threshold. - relation: A string in (`less than`, `at least`), defining the relational - operator for comparison. - Two relational comparisons are supported for now: - if 'less than', the actual number of sentences < the threshold; - if 'at least', the actual number of sentences >= the threshold. - - Returns: - A string representing the instruction description. - """ - # The number of sentences as a threshold for comparison. - self._num_sentences_threshold = num_sentences - if self._num_sentences_threshold is None or self._num_sentences_threshold < 0: - self._num_sentences_threshold = random.randint(1, _MAX_NUM_SENTENCES) - - if relation is None: - self._comparison_relation = random.choice(_COMPARISON_RELATION) - elif relation not in _COMPARISON_RELATION: - raise ValueError( - f"The supported relation for comparison must be in {_COMPARISON_RELATION}, but {relation} is given." - ) - else: - self._comparison_relation = relation - - self._description_pattern = "Your response should contain {relation} {num_sentences} sentences." - return self._description_pattern.format( - relation=self._comparison_relation, - num_sentences=self._num_sentences_threshold, - ) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return { - "num_sentences": self._num_sentences_threshold, - "relation": self._comparison_relation, - } - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["num_sentences", "relation"] - - def check_following(self, value): - """Check if the number of sentences follows the instruction. - - Args: - value: A string representing the response. - - Returns: - True if the response follows the instruction. - - Raise: - ValueError if the string in `instruction_args` is not in - [`less_than`, `at_least`]. - """ - num_sentences = instructions_util.count_sentences(value) - if self._comparison_relation == _COMPARISON_RELATION[0]: - return num_sentences < self._num_sentences_threshold - elif self._comparison_relation == _COMPARISON_RELATION[1]: - return num_sentences >= self._num_sentences_threshold - - -class PlaceholderChecker(Instruction): - """Check the placeholders in template writing.""" - - def build_description(self, *, num_placeholders=None): - """Build the instruction description. - - Args: - num_placeholders: An integer denoting the minimum number of - placeholders required in the response. - - Returns: - A string representing the instruction description. - """ - self._num_placeholders = num_placeholders - if self._num_placeholders is None or self._num_placeholders < 0: - self._num_placeholders = random.randint(1, _NUM_PLACEHOLDERS) - self._description_pattern = ( - "The response must contain at least {num_placeholders} placeholders " - + "represented by square brackets, such as [address]." - ) - return self._description_pattern.format(num_placeholders=self._num_placeholders) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return {"num_placeholders": self._num_placeholders} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["num_placeholders"] - - def check_following(self, value): - """Check if the number of placeholders follows the instruction. - - Args: - value: A string representing the response. - - Returns: - True if the actual number of placeholders in the response is greater than - or equal to `num_placeholders`; otherwise, False. - """ - placeholders = re.findall(r"\[.*?\]", value) - num_placeholders = len(placeholders) - return num_placeholders >= self._num_placeholders - - -class BulletListChecker(Instruction): - """Checks the bullet list in the prompt.""" - - def build_description(self, *, num_bullets=None): - """Build the instruction description. - - Args: - num_bullets: An integer specifying the exact number of bullet lists - that is required to appear in the response. - - Returns: - A string representing the instruction description. - """ - self._num_bullets = num_bullets - if self._num_bullets is None or self._num_bullets < 0: - self._num_bullets = random.randint(1, _NUM_BULLETS) - self._description_pattern = ( - "Your answer must contain exactly {num_bullets} bullet points. " - + "Use the markdown bullet points such as:\n" - + "* This is point 1. \n" - + "* This is point 2" - ) - return self._description_pattern.format(num_bullets=self._num_bullets) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return {"num_bullets": self._num_bullets} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["num_bullets"] - - def check_following(self, value): - r"""Check if the number of bullet lists meets the requirement. - - Args: - value: A string representing the response. The response is expected to - contain some bullet lists that start with `\*`. - - Returns: - True if the actual number of bullet lists in the response meets the - requirement. - """ - bullet_lists = re.findall(r"^\s*\*[^\*].*$", value, flags=re.MULTILINE) - bullet_lists_2 = re.findall(r"^\s*-.*$", value, flags=re.MULTILINE) - num_bullet_lists = len(bullet_lists) + len(bullet_lists_2) - return num_bullet_lists == self._num_bullets - - -class ConstrainedResponseChecker(Instruction): - """Checks the constrained response.""" - - def build_description(self): - """Build the instruction description.""" - # A sequence of string(s) representing the options of the expected response. - self._constrained_responses = _CONSTRAINED_RESPONSE_OPTIONS - self._description_pattern = "Answer with one of the following options: {response_options}" - return self._description_pattern.format(response_options=self._constrained_responses) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response matches the constrained options. - - Args: - value: A string representing the response. - - Returns: - True if the actual response contains one of the options in the constrained - responses; otherwise False. - """ - value = value.strip() - for constrained_response in self._constrained_responses: - if constrained_response in value: - return True - return False - - -class ConstrainedStartChecker(Instruction): - """Checks the response start.""" - - def build_description(self, *, starter=None): - """Build the instruction description. - - Args: - starter: A string representing the keyward that the response should start - with. - - Returns: - A string representing the instruction description. - """ - self._starter = starter.strip() if isinstance(starter, str) else starter - if self._starter is None: - self._starter = random.choice(_STARTER_OPTIONS) - self._description_pattern = ( - "During the conversation, when it is your turn, " + "please always start with {starter}" - ) - return self._description_pattern.format(starter=self._starter) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return {"starter": self._starter} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["starter"] - - def check_following(self, value): - """Checks if the response starts with the constrained keyword or phrase. - - Args: - value: A string representing the response. - - Returns: - True if the response starts with the given phrase or keyword that is - contained in `instruction_args`; otherwise, False. - """ - response_pattern = r"^\s*" + self._starter + r".*$" - response_with_constrained_start = re.search(response_pattern, value, flags=re.MULTILINE) - return True if response_with_constrained_start else False - - -class HighlightSectionChecker(Instruction): - """Checks the highlighted section.""" - - def build_description(self, *, num_highlights=None): - """Build the instruction description. - - Args: - num_highlights: An integer specifying the minimum number of highlighted - sections. - - Returns: - A string representing the instruction description. - """ - self._num_highlights = num_highlights - if self._num_highlights is None or self._num_highlights < 0: - self._num_highlights = random.randint(1, _NUM_HIGHLIGHTED_SECTIONS) - - self._description_pattern = ( - "Highlight at least {num_highlights} sections in your answer with " - + "markdown, i.e. *highlighted section*." - ) - - return self._description_pattern.format(num_highlights=self._num_highlights) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return {"num_highlights": self._num_highlights} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["num_highlights"] - - def check_following(self, value): - """Checks if the number of highlighted sections meets the requirement. - - Args: - value: a string repesenting the response. The response is expected to - contain highlighted sections in the format of *highlighted*. - - Returns: - True if the actual number of highlighted sections in the format of - *highlighed sections* meets the minimum requirement; otherwise False. - """ - num_highlights = 0 - highlights = re.findall(r"\*[^\n\*]*\*", value) - double_highlights = re.findall(r"\*\*[^\n\*]*\*\*", value) - for highlight in highlights: - if highlight.strip("*").strip(): - num_highlights += 1 - for highlight in double_highlights: - if highlight.removeprefix("**").removesuffix("**").strip(): - num_highlights += 1 - - return num_highlights >= self._num_highlights - - -class SectionChecker(Instruction): - """Checks the sections.""" - - def build_description(self, *, section_spliter=None, num_sections=None): - """Build the instruction description. - - Args: - section_spliter: A string represents the section spliter keyword that - marks a new section, i.e., `Section` or `SECTION`. - num_sections: An integer specifying the number of sections. - - Returns: - A string representing the instruction description. - """ - self._section_spliter = section_spliter.strip() if isinstance(section_spliter, str) else section_spliter - if self._section_spliter is None: - self._section_spliter = random.choice(_SECTION_SPLITER) - - self._num_sections = num_sections - if self._num_sections is None or self._num_sections < 0: - self._num_sections = random.randint(1, _NUM_SECTIONS) - - self._description_pattern = ( - "Your response must have {num_sections} sections. Mark the beginning " - + "of each section with {section_spliter} X, such as:\n" - + "{section_spliter} 1\n" - + "[content of section 1]\n" - + "{section_spliter} 2\n" - + "[content of section 2]" - ) - - return self._description_pattern.format(num_sections=self._num_sections, section_spliter=self._section_spliter) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return { - "section_spliter": self._section_spliter, - "num_sections": self._num_sections, - } - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["section_spliter", "num_sections"] - - def check_following(self, value): - """Checks the response contains multiple sections. - - Args: - value: A string representing the response. The response is expected - to contain multiple sections (number of sections is greater than 1). - A new section starts with `Section 1`, where the number denotes the - section index. - - Returns: - True if the number of sections in the response is greater than or equal to - the minimum number of sections; otherwise, False. - """ - section_splitter_patten = r"\s?" + self._section_spliter + r"\s?\d+\s?" - sections = re.split(section_splitter_patten, value) - num_sections = len(sections) - 1 - return num_sections >= self._num_sections - - -class ParagraphChecker(Instruction): - """Checks the paragraphs.""" - - def build_description(self, *, num_paragraphs=None): - """Build the instruction description. - - Args: - num_paragraphs: An integer specifying the number of paragraphs. - - Returns: - A string representing the instruction description. - """ - self._num_paragraphs = num_paragraphs - if self._num_paragraphs is None or self._num_paragraphs < 0: - self._num_paragraphs = random.randint(1, _NUM_PARAGRAPHS) - - self._description_pattern = ( - "There should be {num_paragraphs} paragraphs. " + "Paragraphs are separated with the markdown divider: ***" - ) - - return self._description_pattern.format(num_paragraphs=self._num_paragraphs) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return {"num_paragraphs": self._num_paragraphs} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["num_paragraphs"] - - def check_following(self, value): - """Checks the response contains required number of paragraphs. - - Args: - value: A string representing the response. The response may contain - paragraphs that are separated by the markdown divider: `***`. - - Returns: - True if the actual number of paragraphs is the same as required; - otherwise, False. - """ - paragraphs = re.split(r"\s?\*\*\*\s?", value) - num_paragraphs = len(paragraphs) - - for index, paragraph in enumerate(paragraphs): - if not paragraph.strip(): - if index == 0 or index == len(paragraphs) - 1: - num_paragraphs -= 1 - else: - return False - - return num_paragraphs == self._num_paragraphs - - -class PostscriptChecker(Instruction): - """Checks the postscript.""" - - def build_description(self, *, postscript_marker=None): - """Build the instruction description. - - Args: - postscript_marker: A string containing the keyword that marks the start - of the postscript section. - - Returns: - A string representing the instruction description. - """ - self._postscript_marker = ( - postscript_marker.strip() if isinstance(postscript_marker, str) else postscript_marker - ) - if self._postscript_marker is None: - self._postscript_marker = random.choice(_POSTSCRIPT_MARKER) - - self._description_pattern = ( - "At the end of your response, please explicitly add a postscript " + "starting with {postscript}" - ) - - return self._description_pattern.format(postscript=self._postscript_marker) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return {"postscript_marker": self._postscript_marker} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["postscript_marker"] - - def check_following(self, value): - """Checks if the response follows the postscript format. - - Args: - value: a string representing the response. The response is expected to - contain a postscript section. - - Returns: - True if the response contains a postscript section starting with - the keyword containing in the `instruction_args`; otherwise False. - """ - value = value.lower() - if self._postscript_marker == "P.P.S": - postscript_pattern = r"\s*p\.\s?p\.\s?s.*$" - elif self._postscript_marker == "P.S.": - postscript_pattern = r"\s*p\.\s?s\..*$" - else: - postscript_pattern = r"\s*" + self._postscript_marker.lower() + r".*$" - postscript = re.findall(postscript_pattern, value, flags=re.MULTILINE) - return True if postscript else False - - -class RephraseChecker(Instruction): - """Checks the repharse.""" - - def build_description(self, *, original_message): - """Build the instruction description. - - Args: - original_message: A string representing the original message. The - rephrased response should only change its words/sentences in between - its two asterisks, for example, *change me*. Both original and rephrased - messages should contain the changes in the form of *change me*. - - Returns: - A string representing the instruction description. - """ - if not self.is_change(original_message): - raise ValueError(f"Message {original_message} does not contain changes in the form of *change me*.") - - self._reference_without_change = original_message - self._description = ( - "Rephrasing: Your rephrased response should only" - + "change the words/sentences in between two asterisks" - + "such as *change me*." - ) - return self._description - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return {"original_message": self._reference_without_change} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["original_message"] - - def check_following(self, value): - r"""Checks if the rephrasing follows the instruction. - - Args: - value: A string representing the response, which is expected to rephras - the string of `instruction_args`. - - Returns: - True if `value` and `instruction_args` only differ by the words/sentences - in between two asterisks such as *change me*; otherwise, False. - """ - if not self.is_change(value): - raise ValueError(f"value {value} does not contain changes in the form of *change me*.") - - response_without_changes = self.strip_changes(value) - reference_without_changes = self.strip_changes(self._reference_without_change) - - return response_without_changes == reference_without_changes - - def is_change(self, response): - """Check if there is change in the response in the form of *change me*.""" - return re.search(r"\*.*\*", response) - - def strip_changes(self, response): - """Strips off the changes.""" - return re.sub(r"\*.*\*", "", response) - - -class KeywordChecker(Instruction): - """Check the exisitence of certain keywords.""" - - def build_description(self, *, keywords=None): - """Build the instruction description. - - Args: - keywords: A sequence of strings representing the keywords that are - expected in the response. - - Returns: - A string representing the instruction description. - """ - if not keywords: - self._keywords = instructions_util.generate_keywords(num_keywords=_NUM_KEYWORDS) - else: - self._keywords = keywords - self._keywords = sorted(self._keywords) - - self._description_pattern = "Include keywords {keywords} in the response." - - return self._description_pattern.format(keywords=self._keywords) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return {"keywords": self._keywords} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["keywords"] - - def check_following(self, value): - """Check if the response contain the expected keywords. - - Returns: - bool: True if the response contains all expected keywords, False otherwise - """ - for keyword in self._keywords: - if not re.search(keyword, value, flags=re.IGNORECASE): - return False - return True - - -class KeywordFrequencyChecker(Instruction): - """Check the keyword frequency.""" - - def build_description(self, *, keyword=None, frequency=None, relation=None): - """Build the instruction description. - - Args: - keyword: A string representing a keyword that is expected in the response. - frequency: An integer specifying the number of times `keyword` is expected - to appear in the response. - relation: A string in (`less than`, `at least`), defining the relational - operator for comparison. - Two relational comparisons are supported for now: - if 'less than', the actual number of occurrences < frequency; - if 'at least', the actual number of occurrences >= frequency. - - Returns: - A string representing the instruction description. - """ - if not keyword: - self._keyword = instructions_util.generate_keywords(num_keywords=1)[0] - else: - self._keyword = keyword.strip() - - self._frequency = frequency - if self._frequency is None or self._frequency < 0: - self._frequency = random.randint(1, _KEYWORD_FREQUENCY) - - if relation is None: - self._comparison_relation = random.choice(_COMPARISON_RELATION) - elif relation not in _COMPARISON_RELATION: - raise ValueError( - f"The supported relation for comparison must be in {_COMPARISON_RELATION}, but {relation} is given." - ) - else: - self._comparison_relation = relation - - self._description_pattern = ( - "In your response, the word {keyword} should appear {relation} " + "{frequency} times." - ) - - return self._description_pattern.format( - keyword=self._keyword, - relation=self._comparison_relation, - frequency=self._frequency, - ) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return { - "keyword": self._keyword, - "frequency": self._frequency, - "relation": self._comparison_relation, - } - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["keyword", "frequency", "relation"] - - def check_following(self, value): - """Checks if the response contain the keyword with required frequency.""" - actual_occurrences = len(re.findall(self._keyword, value, flags=re.IGNORECASE)) - - if self._comparison_relation == _COMPARISON_RELATION[0]: - return actual_occurrences < self._frequency - elif self._comparison_relation == _COMPARISON_RELATION[1]: - return actual_occurrences >= self._frequency - - -class NumberOfWords(Instruction): - """Checks the number of words.""" - - def build_description(self, *, num_words=None, relation=None): - """Build the instruction description. - - Args: - num_words: An integer specifying the number of words contained in the - response. - relation: A string in (`less than`, `at least`), defining the relational - operator for comparison. - Two relational comparisons are supported for now: - if 'less than', the actual number of words < num_words; - if 'at least', the actual number of words >= num_words. - - Returns: - A string representing the instruction description. - """ - self._num_words = num_words - if self._num_words is None or self._num_words < 0: - self._num_words = random.randint(_NUM_WORDS_LOWER_LIMIT, _NUM_WORDS_UPPER_LIMIT) - - if relation is None: - self._comparison_relation = random.choice(_COMPARISON_RELATION) - elif relation not in _COMPARISON_RELATION: - raise ValueError( - f"The supported relation for comparison must be in {_COMPARISON_RELATION}, but {relation} is given." - ) - else: - self._comparison_relation = relation - - self._description_pattern = "Answer with {relation} {num_words} words." - - return self._description_pattern.format(relation=self._comparison_relation, num_words=self._num_words) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return {"num_words": self._num_words, "relation": self._comparison_relation} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["num_words", "relation"] - - def check_following(self, value): - """Checks if the response contains the expected number of words.""" - num_words = instructions_util.count_words(value) - - if self._comparison_relation == _COMPARISON_RELATION[0]: - return num_words < self._num_words - elif self._comparison_relation == _COMPARISON_RELATION[1]: - return num_words >= self._num_words - - -class JsonFormat(Instruction): - """Check the Json format.""" - - def build_description(self): - self._description_pattern = ( - "Entire output should be wrapped in JSON format. You can use markdown ticks such as ```." - ) - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - value = ( - value.strip() - .removeprefix("```json") - .removeprefix("```Json") - .removeprefix("```JSON") - .removeprefix("```") - .removesuffix("```") - .strip() - ) - try: - json.loads(value) - except ValueError: - return False - return True - - -class ParagraphFirstWordCheck(Instruction): - """Check the paragraph and the first word of the nth paragraph.""" - - def build_description(self, num_paragraphs=None, nth_paragraph=None, first_word=None): - r"""Build the instruction description. - - Args: - num_paragraphs: An integer indicating the number of paragraphs expected - in the response. A paragraph is a subset of the string that is - expected to be separated by '\n\n'. - nth_paragraph: An integer indicating the paragraph number that we look at. - Note that n starts from 1. - first_word: A string that represent the first word of the bth paragraph. - - Returns: - A string representing the instruction description. - """ - self._num_paragraphs = num_paragraphs - if self._num_paragraphs is None or self._num_paragraphs < 0: - self._num_paragraphs = random.randint(1, _NUM_PARAGRAPHS) - - self._nth_paragraph = nth_paragraph - if self._nth_paragraph is None or self._nth_paragraph <= 0 or self._nth_paragraph > self._num_paragraphs: - self._nth_paragraph = random.randint(1, self._num_paragraphs + 1) - - self._first_word = first_word - if self._first_word is None: - self._first_word = instructions_util.generate_keywords(num_keywords=1)[0] - self._first_word = self._first_word.lower() - - self._description_pattern = ( - "There should be {num_paragraphs} paragraphs. " - + "Paragraphs and only paragraphs are separated with each other by two " - + "new lines as if it was '\\n\\n' in python. " - + "Paragraph {nth_paragraph} must start with word {first_word}." - ) - - return self._description_pattern.format( - num_paragraphs=self._num_paragraphs, - nth_paragraph=self._nth_paragraph, - first_word=self._first_word, - ) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return { - "num_paragraphs": self._num_paragraphs, - "nth_paragraph": self._nth_paragraph, - "first_word": self._first_word, - } - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["num_paragraphs", "nth_paragraph", "first_word"] - - def check_following(self, value): - """Checks for required number of paragraphs and correct first word. - - Args: - value: a string representing the response. The response may contain - paragraphs that are separated by two new lines and the first word of - the nth paragraph will have to match a specified word. - - Returns: - True if the number of paragraphs is the same as required and the first - word of the specified paragraph is the same as required. Otherwise, false. - """ - paragraphs = re.split(r"\n\n", value) - num_paragraphs = len(paragraphs) - - for paragraph in paragraphs: - if not paragraph.strip(): - num_paragraphs -= 1 - - # check that index doesn't go out of bounds - if self._nth_paragraph <= num_paragraphs: - paragraph = paragraphs[self._nth_paragraph - 1].strip() - if not paragraph: - return False - else: - return False - - first_word = "" - punctuation = {".", ",", "?", "!", "'", '"'} - - # get first word and remove punctuation - word = paragraph.split()[0].strip() - # TODO(jeffrey): make more complex? - word = word.lstrip("'") - word = word.lstrip('"') - - for letter in word: - if letter in punctuation: - break - first_word += letter.lower() - - return num_paragraphs == self._num_paragraphs and first_word == self._first_word - - -# TODO(jeffrey) add relation - at least/at most? -class KeySentenceChecker(Instruction): - """Check the existence of certain key sentences.""" - - def build_description(self, key_sentences=None, num_sentences=None): - """Build the instruction description. - - Args: - key_sentences: A sequences of strings representing the key sentences that - are expected in the response. - num_sentences: The number of key sentences that are expected to be seen in - the response. - - Returns: - A string representing the instruction description. - """ - if not key_sentences: - # TODO(jeffrey) make a generate sentences function? wonderwords package - self._key_sentences = set("For now, this is fine.") - else: - self._key_sentences = key_sentences - - if not num_sentences: - self._num_sentences = random.randint(1, len(self._key_sentences)) - else: - self._num_sentences = num_sentences - - self._description_pattern = "Include {num_sentences} of the following sentences {key_sentences}" - - return self._description_pattern.format(num_sentences=self._num_sentences, key_sentences=self._key_sentences) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return { - "num_sentences": self._num_sentences, - "key_sentences": list(self._key_sentences), - } - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["num_sentences", "key_sentences"] - - def check_following(self, value): - """Checks if the response contains the expected key sentences.""" - count = 0 - sentences = instructions_util.split_into_sentences(value) - for sentence in self._key_sentences: - if sentence in sentences: - count += 1 - - return count == self._num_sentences - - -class ForbiddenWords(Instruction): - """Checks that specified words are not used in response.""" - - def build_description(self, forbidden_words=None): - """Build the instruction description. - - Args: - forbidden_words: A sequences of strings respresenting words that are not - allowed in the response. - - Returns: - A string representing the instruction description. - """ - if not forbidden_words: - self._forbidden_words = instructions_util.generate_keywords(num_keywords=_NUM_KEYWORDS) - else: - self._forbidden_words = list(set(forbidden_words)) - self._forbidden_words = sorted(self._forbidden_words) - self._description_pattern = "Do not include keywords {forbidden_words} in the response." - - return self._description_pattern.format(forbidden_words=self._forbidden_words) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return {"forbidden_words": self._forbidden_words} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["forbidden_words"] - - def check_following(self, value): - """Check if the response does not contain the expected keywords.""" - for word in self._forbidden_words: - if re.search(r"\b" + word + r"\b", value, flags=re.IGNORECASE): - return False - return True - - -class RephraseParagraph(Instruction): - """Checks that the paragraph is rephrased.""" - - def build_description(self, *, original_paragraph, low, high): - """Builds the instruction description. - - Args: - original_paragraph: A string presenting the original paragraph. The - rephrases response should have betweeb low-high words in common. - low: An integer presenting the lower bound of similar words. - high: An integer representing the upper bound of similar words. - - Returns: - A string representing the instruction description. - """ - # TODO(jeffrey) make more encompassing - self._original_paragraph = original_paragraph - self._low = low - self._high = high - - self._description = ( - "Rephrase the following paragraph: " - + "{original_paragraph}\nYour response should have " - + "between {low} and {high} of the same words. " - + "Words are the same if and only if all of the " - + "letters, ignoring cases, are the same. For " - + "example, 'run' is the same as 'Run' but different " - + "to 'ran'." - ) - - return self._description.format(original_paragraph=original_paragraph, low=self._low, high=self._high) - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return { - "original_paragraph": self._original_paragraph, - "low": self._low, - "high": self._high, - } - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["original_paragraph", "low", "high"] - - def check_following(self, value): - val_words = re.findall(r"\w+", value.lower()) - original_words = re.findall(r"\w+", self._original_paragraph.lower()) - similar_words = 0 - - dict_val = collections.Counter(val_words) - dict_original = collections.Counter(original_words) - - for word in dict_original: - similar_words += min(dict_original[word], dict_val[word]) - - return similar_words >= self._low and similar_words <= self._high - - -class TwoResponsesChecker(Instruction): - """Check that two responses were given.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = ( - "Give two different responses. Responses and only responses should" - " be separated by 6 asterisk symbols: ******." - ) - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyward args of `build_description`.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response has two different answers. - - Args: - value: A string representing the response. - - Returns: - True if two responses are detected and false otherwise. - """ - valid_responses = [] - responses = value.split("******") - for index, response in enumerate(responses): - if not response.strip(): - if index != 0 and index != len(responses) - 1: - return False - else: - valid_responses.append(response) - return len(valid_responses) == 2 and valid_responses[0].strip() != valid_responses[1].strip() - - -class RepeatPromptThenAnswer(Instruction): - """Checks that Prompt is first repeated then answered.""" - - def build_description(self, *, prompt_to_repeat=None): - """Build the instruction description. - - Args: - prompt_to_repeat: The prompt that is meant to be repeated. - - Returns: - A string representing the instruction description. - """ - if not prompt_to_repeat: - raise ValueError("prompt_to_repeat must be set.") - else: - self._prompt_to_repeat = prompt_to_repeat - self._description_pattern = ( - "First repeat the request word for word without change," - " then give your answer (1. do not say any words or characters" - " before repeating the request; 2. the request you need to repeat" - " does not include this sentence)" - ) - return self._description_pattern - - def get_instruction_args(self): - return {"prompt_to_repeat": self._prompt_to_repeat} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["prompt_to_repeat"] - - def check_following(self, value): - if value.strip().lower().startswith(self._prompt_to_repeat.strip().lower()): - return True - return False - - -class EndChecker(Instruction): - """Checks that the prompt ends with a given phrase.""" - - def build_description(self, *, end_phrase=None): - """Build the instruction description. - - Args: - end_phrase: A string representing the phrase the response should end with. - - Returns: - A string representing the instruction description. - """ - self._end_phrase = end_phrase.strip() if isinstance(end_phrase, str) else end_phrase - if self._end_phrase is None: - self._end_phrase = random.choice(_ENDING_OPTIONS) - self._description_pattern = ( - "Finish your response with this exact phrase {ender}. No other words should follow this phrase." - ) - return self._description_pattern.format(ender=self._end_phrase) - - def get_instruction_args(self): - return {"end_phrase": self._end_phrase} - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["end_phrase"] - - def check_following(self, value): - """Checks if the response ends with the expected phrase.""" - value = value.strip().strip('"').lower() - self._end_phrase = self._end_phrase.strip().lower() - return value.endswith(self._end_phrase) - - -class TitleChecker(Instruction): - """Checks the response for a title.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = ( - "Your answer must contain a title, wrapped in double angular brackets, such as <>." - ) - return self._description_pattern - - def get_instruction_args(self): - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response contains a title.""" - pattern = r"<<[^\n]+>>" - re_pattern = re.compile(pattern) - titles = re.findall(re_pattern, value) - - for title in titles: - if title.lstrip("<").rstrip(">").strip(): - return True - return False - - -class LetterFrequencyChecker(Instruction): - """Checks letter frequency.""" - - def build_description(self, *, letter=None, let_frequency=None, let_relation=None): - """Build the instruction description. - - Args: - letter: A string representing a letter that is expected in the response. - let_frequency: An integer specifying the number of times `keyword` is - expected to appear in the response. - let_relation: A string in (`less than`, `at least`), defining the - relational operator for comparison. Two relational comparisons are - supported for now; if 'less than', the actual number of - occurrences < frequency; if 'at least', the actual number of - occurrences >= frequency. - - Returns: - A string representing the instruction description. - """ - if not letter or len(letter) > 1 or ord(letter.lower()) < 97 or ord(letter.lower()) > 122: - self._letter = random.choice(list(string.ascii_letters)) - else: - self._letter = letter.strip() - self._letter = self._letter.lower() - - self._frequency = let_frequency - if self._frequency is None or self._frequency < 0: - self._frequency = random.randint(1, _LETTER_FREQUENCY) - - if let_relation is None: - self._comparison_relation = random.choice(_COMPARISON_RELATION) - elif let_relation not in _COMPARISON_RELATION: - raise ValueError( - "The supported relation for comparison must be in " - f"{_COMPARISON_RELATION}, but {let_relation} is given." - ) - else: - self._comparison_relation = let_relation - - self._description_pattern = ( - "In your response, the letter {letter} should appear {let_relation} {let_frequency} times." - ) - - return self._description_pattern.format( - letter=self._letter, - let_frequency=self._frequency, - let_relation=self._comparison_relation, - ) - - def get_instruction_args(self): - """Returns the keyword args of build description.""" - return { - "letter": self._letter, - "let_frequency": self._frequency, - "let_relation": self._comparison_relation, - } - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["letter", "let_frequency", "let_relation"] - - def check_following(self, value): - """Checks that the response contains the letter at the right frequency.""" - value = value.lower() - letters = collections.Counter(value) - - if self._comparison_relation == _COMPARISON_RELATION[0]: - return letters[self._letter] < self._frequency - else: - return letters[self._letter] >= self._frequency - - -class CapitalLettersEnglishChecker(Instruction): - """Checks that the response is in english and is in all capital letters.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Your entire response should be in English, and in all capital letters." - return self._description_pattern - - def get_instruction_args(self): - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks that the response is in English and in all capital letters.""" - assert isinstance(value, str) - - try: - return value.isupper() and langdetect.detect(value) == "en" - except langdetect.LangDetectException as e: - # Count as instruction is followed. - logging.error("Unable to detect language for text %s due to %s", value, e) # refex: disable=pytotw.037 - return True - - -class LowercaseLettersEnglishChecker(Instruction): - """Checks that the response is in english and is in all lowercase letters.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = ( - "Your entire response should be in English, and in all lowercase letters. No capital letters are allowed." - ) - return self._description_pattern - - def get_instruction_args(self): - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks that the response is in English and in all lowercase letters.""" - assert isinstance(value, str) - - try: - return value.islower() and langdetect.detect(value) == "en" - except langdetect.LangDetectException as e: - # Count as instruction is followed. - logging.error("Unable to detect language for text %s due to %s", value, e) # refex: disable=pytotw.037 - return True - - -class CommaChecker(Instruction): - """Checks the response for no commas.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "In your entire response, refrain from the use of any commas." - return self._description_pattern - - def get_instruction_args(self): - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks that the response does not contain commas.""" - return not re.search(r"\,", value) - - -class CapitalWordFrequencyChecker(Instruction): - """Checks frequency of words with all capital letters.""" - - def build_description( - self, - capital_frequency=None, - capital_relation=None, - ): - """Build the instruction description. - - Args: - capital_frequency: An integer that represents the number of words that - should be in all capital letters. - capital_relation: A string that is 'at least' or 'at most' that refers to - the frequency. - - Returns: - A string representing the instruction description. - """ - self._frequency = capital_frequency - if self._frequency is None: - self._frequency = random.randint(1, _ALL_CAPITAL_WORD_FREQUENCY) - - self._comparison_relation = capital_relation - if capital_relation is None: - self._comparison_relation = random.choice(_COMPARISON_RELATION) - elif capital_relation not in _COMPARISON_RELATION: - raise ValueError( - "The supported relation for comparison must be in " - f"{_COMPARISON_RELATION}, but {capital_relation} is given." - ) - - self._description_pattern = ( - "In your response, words with all capital letters should appear {relation} {frequency} times." - ) - - return self._description_pattern.format(frequency=self._frequency, relation=self._comparison_relation) - - def get_instruction_args(self): - """Returns the keyword args of build description.""" - return { - "capital_frequency": self._frequency, - "capital_relation": self._comparison_relation, - } - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return ["capital_frequency", "capital_relation"] - - def check_following(self, value): - """Checks the frequency of words with all capital letters.""" - # Hyphenated words will count as one word - words = instructions_util.nltk.word_tokenize(value) - capital_words = [word for word in words if word.isupper()] - - capital_words = len(capital_words) - - if self._comparison_relation == _COMPARISON_RELATION[0]: - return capital_words < self._frequency - else: - return capital_words >= self._frequency - - -class QuotationChecker(Instruction): - """Checks response is wrapped with double quotation marks.""" - - def build_description(self): - """Build the instruction description.""" - self._description_pattern = "Wrap your entire response with double quotation marks." - return self._description_pattern - - def get_instruction_args(self): - """Returns the keyword args of build description.""" - return None - - def get_instruction_args_keys(self): - """Returns the args keys of `build_description`.""" - return [] - - def check_following(self, value): - """Checks if the response is wrapped with double quotation marks.""" - value = value.strip() - return len(value) > 1 and value[0] == '"' and value[-1] == '"' diff --git a/src/lighteval/tasks/tasks/ifeval/instructions_registry.py b/src/lighteval/tasks/tasks/ifeval/instructions_registry.py deleted file mode 100644 index 4dada73d4..000000000 --- a/src/lighteval/tasks/tasks/ifeval/instructions_registry.py +++ /dev/null @@ -1,168 +0,0 @@ -# Copyright 2023 The Google Research Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Registry of all instructions.""" - -import lighteval.tasks.tasks.ifeval.instructions as instructions - - -_KEYWORD = "keywords:" - -_LANGUAGE = "language:" - -_LENGTH = "length_constraints:" - -_CONTENT = "detectable_content:" - -_FORMAT = "detectable_format:" - -_MULTITURN = "multi-turn:" - -_COMBINATION = "combination:" - -_STARTEND = "startend:" - -_CHANGE_CASES = "change_case:" - -_PUNCTUATION = "punctuation:" - -INSTRUCTION_DICT = { - _KEYWORD + "existence": instructions.KeywordChecker, - _KEYWORD + "frequency": instructions.KeywordFrequencyChecker, - # TODO(jeffreyzhou): make a proper set of sentences to choose from - # _KEYWORD + "key_sentences": instructions.KeySentenceChecker, - _KEYWORD + "forbidden_words": instructions.ForbiddenWords, - _KEYWORD + "letter_frequency": instructions.LetterFrequencyChecker, - _LANGUAGE + "response_language": instructions.ResponseLanguageChecker, - _LENGTH + "number_sentences": instructions.NumberOfSentences, - _LENGTH + "number_paragraphs": instructions.ParagraphChecker, - _LENGTH + "number_words": instructions.NumberOfWords, - _LENGTH + "nth_paragraph_first_word": instructions.ParagraphFirstWordCheck, - _CONTENT + "number_placeholders": instructions.PlaceholderChecker, - _CONTENT + "postscript": instructions.PostscriptChecker, - _FORMAT + "number_bullet_lists": instructions.BulletListChecker, - # TODO(jeffreyzhou): Pre-create paragraph or use prompt to replace - # _CONTENT + "rephrase_paragraph": instructions.RephraseParagraph, - _FORMAT + "constrained_response": instructions.ConstrainedResponseChecker, - _FORMAT + "number_highlighted_sections": (instructions.HighlightSectionChecker), - _FORMAT + "multiple_sections": instructions.SectionChecker, - # TODO(tianjianlu): Re-enable rephrasing with preprocessing the message. - # _FORMAT + "rephrase": instructions.RephraseChecker, - _FORMAT + "json_format": instructions.JsonFormat, - _FORMAT + "title": instructions.TitleChecker, - # TODO(tianjianlu): Re-enable with specific prompts. - # _MULTITURN + "constrained_start": instructions.ConstrainedStartChecker, - _COMBINATION + "two_responses": instructions.TwoResponsesChecker, - _COMBINATION + "repeat_prompt": instructions.RepeatPromptThenAnswer, - _STARTEND + "end_checker": instructions.EndChecker, - _CHANGE_CASES + "capital_word_frequency": instructions.CapitalWordFrequencyChecker, - _CHANGE_CASES + "english_capital": instructions.CapitalLettersEnglishChecker, - _CHANGE_CASES + "english_lowercase": instructions.LowercaseLettersEnglishChecker, - _PUNCTUATION + "no_comma": instructions.CommaChecker, - _STARTEND + "quotation": instructions.QuotationChecker, -} - -INSTRUCTION_CONFLICTS = { - _KEYWORD + "existence": {_KEYWORD + "existence"}, - _KEYWORD + "frequency": {_KEYWORD + "frequency"}, - # TODO(jeffreyzhou): make a proper set of sentences to choose from - # _KEYWORD + "key_sentences": instructions.KeySentenceChecker, - _KEYWORD + "forbidden_words": {_KEYWORD + "forbidden_words"}, - _KEYWORD + "letter_frequency": {_KEYWORD + "letter_frequency"}, - _LANGUAGE + "response_language": { - _LANGUAGE + "response_language", - _FORMAT + "multiple_sections", - _KEYWORD + "existence", - _KEYWORD + "frequency", - _KEYWORD + "forbidden_words", - _STARTEND + "end_checker", - _CHANGE_CASES + "english_capital", - _CHANGE_CASES + "english_lowercase", - }, - _LENGTH + "number_sentences": {_LENGTH + "number_sentences"}, - _LENGTH + "number_paragraphs": { - _LENGTH + "number_paragraphs", - _LENGTH + "nth_paragraph_first_word", - _LENGTH + "number_sentences", - _LENGTH + "nth_paragraph_first_word", - }, - _LENGTH + "number_words": {_LENGTH + "number_words"}, - _LENGTH + "nth_paragraph_first_word": { - _LENGTH + "nth_paragraph_first_word", - _LENGTH + "number_paragraphs", - }, - _CONTENT + "number_placeholders": {_CONTENT + "number_placeholders"}, - _CONTENT + "postscript": {_CONTENT + "postscript"}, - _FORMAT + "number_bullet_lists": {_FORMAT + "number_bullet_lists"}, - # TODO(jeffreyzhou): Pre-create paragraph or use prompt to replace - # _CONTENT + "rephrase_paragraph": instructions.RephraseParagraph, - _FORMAT + "constrained_response": set(INSTRUCTION_DICT.keys()), - _FORMAT + "number_highlighted_sections": {_FORMAT + "number_highlighted_sections"}, - _FORMAT + "multiple_sections": { - _FORMAT + "multiple_sections", - _LANGUAGE + "response_language", - _FORMAT + "number_highlighted_sections", - }, - # TODO(tianjianlu): Re-enable rephrasing with preprocessing the message. - # _FORMAT + "rephrase": instructions.RephraseChecker, - _FORMAT + "json_format": set(INSTRUCTION_DICT.keys()).difference( - {_KEYWORD + "forbidden_words", _KEYWORD + "existence"} - ), - _FORMAT + "title": {_FORMAT + "title"}, - # TODO(tianjianlu): Re-enable with specific prompts. - # _MULTITURN + "constrained_start": instructions.ConstrainedStartChecker, - _COMBINATION + "two_responses": set(INSTRUCTION_DICT.keys()).difference( - { - _KEYWORD + "forbidden_words", - _KEYWORD + "existence", - _LANGUAGE + "response_language", - _FORMAT + "title", - _PUNCTUATION + "no_comma", - } - ), - _COMBINATION + "repeat_prompt": set(INSTRUCTION_DICT.keys()).difference( - {_KEYWORD + "existence", _FORMAT + "title", _PUNCTUATION + "no_comma"} - ), - _STARTEND + "end_checker": {_STARTEND + "end_checker"}, - _CHANGE_CASES + "capital_word_frequency": { - _CHANGE_CASES + "capital_word_frequency", - _CHANGE_CASES + "english_lowercase", - _CHANGE_CASES + "english_capital", - }, - _CHANGE_CASES + "english_capital": {_CHANGE_CASES + "english_capital"}, - _CHANGE_CASES + "english_lowercase": { - _CHANGE_CASES + "english_lowercase", - _CHANGE_CASES + "english_capital", - }, - _PUNCTUATION + "no_comma": {_PUNCTUATION + "no_comma"}, - _STARTEND + "quotation": {_STARTEND + "quotation", _FORMAT + "title"}, -} - - -def conflict_make(conflicts): - """Makes sure if A conflicts with B, B will conflict with A. - - Args: - conflicts: Dictionary of potential conflicts where key is instruction id - and value is set of instruction ids that it conflicts with. - - Returns: - Revised version of the dictionary. All instructions conflict with - themselves. If A conflicts with B, B will conflict with A. - """ - for key in conflicts: - for k in conflicts[key]: - conflicts[k].add(key) - conflicts[key].add(key) - return conflicts diff --git a/src/lighteval/tasks/tasks/ifeval/instructions_utils.py b/src/lighteval/tasks/tasks/ifeval/instructions_utils.py deleted file mode 100644 index a80fd1a37..000000000 --- a/src/lighteval/tasks/tasks/ifeval/instructions_utils.py +++ /dev/null @@ -1,1708 +0,0 @@ -# Copyright 2023 The Google Research Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Utility library of instructions.""" - -import functools -import random -import re - -import nltk - - -def download_nltk_resources(): - """Download 'punkt' if not already installed""" - try: - nltk.data.find("tokenizers/punkt") - except LookupError: - nltk.download("punkt") - - try: - nltk.data.find("tokenizers/punkt_tab") - except LookupError: - nltk.download("punkt_tab") - - -download_nltk_resources() - -WORD_LIST = [ - "western", - "sentence", - "signal", - "dump", - "spot", - "opposite", - "bottom", - "potato", - "administration", - "working", - "welcome", - "morning", - "good", - "agency", - "primary", - "wish", - "responsibility", - "press", - "problem", - "president", - "steal", - "brush", - "read", - "type", - "beat", - "trainer", - "growth", - "lock", - "bone", - "case", - "equal", - "comfortable", - "region", - "replacement", - "performance", - "mate", - "walk", - "medicine", - "film", - "thing", - "rock", - "tap", - "total", - "competition", - "ease", - "south", - "establishment", - "gather", - "parking", - "world", - "plenty", - "breath", - "claim", - "alcohol", - "trade", - "dear", - "highlight", - "street", - "matter", - "decision", - "mess", - "agreement", - "studio", - "coach", - "assist", - "brain", - "wing", - "style", - "private", - "top", - "brown", - "leg", - "buy", - "procedure", - "method", - "speed", - "high", - "company", - "valuable", - "pie", - "analyst", - "session", - "pattern", - "district", - "pleasure", - "dinner", - "swimming", - "joke", - "order", - "plate", - "department", - "motor", - "cell", - "spend", - "cabinet", - "difference", - "power", - "examination", - "engine", - "horse", - "dimension", - "pay", - "toe", - "curve", - "literature", - "bother", - "fire", - "possibility", - "debate", - "activity", - "passage", - "hello", - "cycle", - "background", - "quiet", - "author", - "effect", - "actor", - "page", - "bicycle", - "error", - "throat", - "attack", - "character", - "phone", - "tea", - "increase", - "outcome", - "file", - "specific", - "inspector", - "internal", - "potential", - "staff", - "building", - "employer", - "shoe", - "hand", - "direction", - "garden", - "purchase", - "interview", - "study", - "recognition", - "member", - "spiritual", - "oven", - "sandwich", - "weird", - "passenger", - "particular", - "response", - "reaction", - "size", - "variation", - "a", - "cancel", - "candy", - "exit", - "guest", - "condition", - "fly", - "price", - "weakness", - "convert", - "hotel", - "great", - "mouth", - "mind", - "song", - "sugar", - "suspect", - "telephone", - "ear", - "roof", - "paint", - "refrigerator", - "organization", - "jury", - "reward", - "engineering", - "day", - "possession", - "crew", - "bar", - "road", - "description", - "celebration", - "score", - "mark", - "letter", - "shower", - "suggestion", - "sir", - "luck", - "national", - "progress", - "hall", - "stroke", - "theory", - "offer", - "story", - "tax", - "definition", - "history", - "ride", - "medium", - "opening", - "glass", - "elevator", - "stomach", - "question", - "ability", - "leading", - "village", - "computer", - "city", - "grand", - "confidence", - "candle", - "priest", - "recommendation", - "point", - "necessary", - "body", - "desk", - "secret", - "horror", - "noise", - "culture", - "warning", - "water", - "round", - "diet", - "flower", - "bus", - "tough", - "permission", - "week", - "prompt", - "connection", - "abuse", - "height", - "save", - "corner", - "border", - "stress", - "drive", - "stop", - "rip", - "meal", - "listen", - "confusion", - "girlfriend", - "living", - "relation", - "significance", - "plan", - "creative", - "atmosphere", - "blame", - "invite", - "housing", - "paper", - "drink", - "roll", - "silver", - "drunk", - "age", - "damage", - "smoke", - "environment", - "pack", - "savings", - "influence", - "tourist", - "rain", - "post", - "sign", - "grandmother", - "run", - "profit", - "push", - "clerk", - "final", - "wine", - "swim", - "pause", - "stuff", - "singer", - "funeral", - "average", - "source", - "scene", - "tradition", - "personal", - "snow", - "nobody", - "distance", - "sort", - "sensitive", - "animal", - "major", - "negotiation", - "click", - "mood", - "period", - "arrival", - "expression", - "holiday", - "repeat", - "dust", - "closet", - "gold", - "bad", - "sail", - "combination", - "clothes", - "emphasis", - "duty", - "black", - "step", - "school", - "jump", - "document", - "professional", - "lip", - "chemical", - "front", - "wake", - "while", - "inside", - "watch", - "row", - "subject", - "penalty", - "balance", - "possible", - "adult", - "aside", - "sample", - "appeal", - "wedding", - "depth", - "king", - "award", - "wife", - "blow", - "site", - "camp", - "music", - "safe", - "gift", - "fault", - "guess", - "act", - "shame", - "drama", - "capital", - "exam", - "stupid", - "record", - "sound", - "swing", - "novel", - "minimum", - "ratio", - "machine", - "shape", - "lead", - "operation", - "salary", - "cloud", - "affair", - "hit", - "chapter", - "stage", - "quantity", - "access", - "army", - "chain", - "traffic", - "kick", - "analysis", - "airport", - "time", - "vacation", - "philosophy", - "ball", - "chest", - "thanks", - "place", - "mountain", - "advertising", - "red", - "past", - "rent", - "return", - "tour", - "house", - "construction", - "net", - "native", - "war", - "figure", - "fee", - "spray", - "user", - "dirt", - "shot", - "task", - "stick", - "friend", - "software", - "promotion", - "interaction", - "surround", - "block", - "purpose", - "practice", - "conflict", - "routine", - "requirement", - "bonus", - "hole", - "state", - "junior", - "sweet", - "catch", - "tear", - "fold", - "wall", - "editor", - "life", - "position", - "pound", - "respect", - "bathroom", - "coat", - "script", - "job", - "teach", - "birth", - "view", - "resolve", - "theme", - "employee", - "doubt", - "market", - "education", - "serve", - "recover", - "tone", - "harm", - "miss", - "union", - "understanding", - "cow", - "river", - "association", - "concept", - "training", - "recipe", - "relationship", - "reserve", - "depression", - "proof", - "hair", - "revenue", - "independent", - "lift", - "assignment", - "temporary", - "amount", - "loss", - "edge", - "track", - "check", - "rope", - "estimate", - "pollution", - "stable", - "message", - "delivery", - "perspective", - "mirror", - "assistant", - "representative", - "witness", - "nature", - "judge", - "fruit", - "tip", - "devil", - "town", - "emergency", - "upper", - "drop", - "stay", - "human", - "neck", - "speaker", - "network", - "sing", - "resist", - "league", - "trip", - "signature", - "lawyer", - "importance", - "gas", - "choice", - "engineer", - "success", - "part", - "external", - "worker", - "simple", - "quarter", - "student", - "heart", - "pass", - "spite", - "shift", - "rough", - "lady", - "grass", - "community", - "garage", - "youth", - "standard", - "skirt", - "promise", - "blind", - "television", - "disease", - "commission", - "positive", - "energy", - "calm", - "presence", - "tune", - "basis", - "preference", - "head", - "common", - "cut", - "somewhere", - "presentation", - "current", - "thought", - "revolution", - "effort", - "master", - "implement", - "republic", - "floor", - "principle", - "stranger", - "shoulder", - "grade", - "button", - "tennis", - "police", - "collection", - "account", - "register", - "glove", - "divide", - "professor", - "chair", - "priority", - "combine", - "peace", - "extension", - "maybe", - "evening", - "frame", - "sister", - "wave", - "code", - "application", - "mouse", - "match", - "counter", - "bottle", - "half", - "cheek", - "resolution", - "back", - "knowledge", - "make", - "discussion", - "screw", - "length", - "accident", - "battle", - "dress", - "knee", - "log", - "package", - "it", - "turn", - "hearing", - "newspaper", - "layer", - "wealth", - "profile", - "imagination", - "answer", - "weekend", - "teacher", - "appearance", - "meet", - "bike", - "rise", - "belt", - "crash", - "bowl", - "equivalent", - "support", - "image", - "poem", - "risk", - "excitement", - "remote", - "secretary", - "public", - "produce", - "plane", - "display", - "money", - "sand", - "situation", - "punch", - "customer", - "title", - "shake", - "mortgage", - "option", - "number", - "pop", - "window", - "extent", - "nothing", - "experience", - "opinion", - "departure", - "dance", - "indication", - "boy", - "material", - "band", - "leader", - "sun", - "beautiful", - "muscle", - "farmer", - "variety", - "fat", - "handle", - "director", - "opportunity", - "calendar", - "outside", - "pace", - "bath", - "fish", - "consequence", - "put", - "owner", - "go", - "doctor", - "information", - "share", - "hurt", - "protection", - "career", - "finance", - "force", - "golf", - "garbage", - "aspect", - "kid", - "food", - "boot", - "milk", - "respond", - "objective", - "reality", - "raw", - "ring", - "mall", - "one", - "impact", - "area", - "news", - "international", - "series", - "impress", - "mother", - "shelter", - "strike", - "loan", - "month", - "seat", - "anything", - "entertainment", - "familiar", - "clue", - "year", - "glad", - "supermarket", - "natural", - "god", - "cost", - "conversation", - "tie", - "ruin", - "comfort", - "earth", - "storm", - "percentage", - "assistance", - "budget", - "strength", - "beginning", - "sleep", - "other", - "young", - "unit", - "fill", - "store", - "desire", - "hide", - "value", - "cup", - "maintenance", - "nurse", - "function", - "tower", - "role", - "class", - "camera", - "database", - "panic", - "nation", - "basket", - "ice", - "art", - "spirit", - "chart", - "exchange", - "feedback", - "statement", - "reputation", - "search", - "hunt", - "exercise", - "nasty", - "notice", - "male", - "yard", - "annual", - "collar", - "date", - "platform", - "plant", - "fortune", - "passion", - "friendship", - "spread", - "cancer", - "ticket", - "attitude", - "island", - "active", - "object", - "service", - "buyer", - "bite", - "card", - "face", - "steak", - "proposal", - "patient", - "heat", - "rule", - "resident", - "broad", - "politics", - "west", - "knife", - "expert", - "girl", - "design", - "salt", - "baseball", - "grab", - "inspection", - "cousin", - "couple", - "magazine", - "cook", - "dependent", - "security", - "chicken", - "version", - "currency", - "ladder", - "scheme", - "kitchen", - "employment", - "local", - "attention", - "manager", - "fact", - "cover", - "sad", - "guard", - "relative", - "county", - "rate", - "lunch", - "program", - "initiative", - "gear", - "bridge", - "breast", - "talk", - "dish", - "guarantee", - "beer", - "vehicle", - "reception", - "woman", - "substance", - "copy", - "lecture", - "advantage", - "park", - "cold", - "death", - "mix", - "hold", - "scale", - "tomorrow", - "blood", - "request", - "green", - "cookie", - "church", - "strip", - "forever", - "beyond", - "debt", - "tackle", - "wash", - "following", - "feel", - "maximum", - "sector", - "sea", - "property", - "economics", - "menu", - "bench", - "try", - "language", - "start", - "call", - "solid", - "address", - "income", - "foot", - "senior", - "honey", - "few", - "mixture", - "cash", - "grocery", - "link", - "map", - "form", - "factor", - "pot", - "model", - "writer", - "farm", - "winter", - "skill", - "anywhere", - "birthday", - "policy", - "release", - "husband", - "lab", - "hurry", - "mail", - "equipment", - "sink", - "pair", - "driver", - "consideration", - "leather", - "skin", - "blue", - "boat", - "sale", - "brick", - "two", - "feed", - "square", - "dot", - "rush", - "dream", - "location", - "afternoon", - "manufacturer", - "control", - "occasion", - "trouble", - "introduction", - "advice", - "bet", - "eat", - "kill", - "category", - "manner", - "office", - "estate", - "pride", - "awareness", - "slip", - "crack", - "client", - "nail", - "shoot", - "membership", - "soft", - "anybody", - "web", - "official", - "individual", - "pizza", - "interest", - "bag", - "spell", - "profession", - "queen", - "deal", - "resource", - "ship", - "guy", - "chocolate", - "joint", - "formal", - "upstairs", - "car", - "resort", - "abroad", - "dealer", - "associate", - "finger", - "surgery", - "comment", - "team", - "detail", - "crazy", - "path", - "tale", - "initial", - "arm", - "radio", - "demand", - "single", - "draw", - "yellow", - "contest", - "piece", - "quote", - "pull", - "commercial", - "shirt", - "contribution", - "cream", - "channel", - "suit", - "discipline", - "instruction", - "concert", - "speech", - "low", - "effective", - "hang", - "scratch", - "industry", - "breakfast", - "lay", - "join", - "metal", - "bedroom", - "minute", - "product", - "rest", - "temperature", - "many", - "give", - "argument", - "print", - "purple", - "laugh", - "health", - "credit", - "investment", - "sell", - "setting", - "lesson", - "egg", - "middle", - "marriage", - "level", - "evidence", - "phrase", - "love", - "self", - "benefit", - "guidance", - "affect", - "you", - "dad", - "anxiety", - "special", - "boyfriend", - "test", - "blank", - "payment", - "soup", - "obligation", - "reply", - "smile", - "deep", - "complaint", - "addition", - "review", - "box", - "towel", - "minor", - "fun", - "soil", - "issue", - "cigarette", - "internet", - "gain", - "tell", - "entry", - "spare", - "incident", - "family", - "refuse", - "branch", - "can", - "pen", - "grandfather", - "constant", - "tank", - "uncle", - "climate", - "ground", - "volume", - "communication", - "kind", - "poet", - "child", - "screen", - "mine", - "quit", - "gene", - "lack", - "charity", - "memory", - "tooth", - "fear", - "mention", - "marketing", - "reveal", - "reason", - "court", - "season", - "freedom", - "land", - "sport", - "audience", - "classroom", - "law", - "hook", - "win", - "carry", - "eye", - "smell", - "distribution", - "research", - "country", - "dare", - "hope", - "whereas", - "stretch", - "library", - "if", - "delay", - "college", - "plastic", - "book", - "present", - "use", - "worry", - "champion", - "goal", - "economy", - "march", - "election", - "reflection", - "midnight", - "slide", - "inflation", - "action", - "challenge", - "guitar", - "coast", - "apple", - "campaign", - "field", - "jacket", - "sense", - "way", - "visual", - "remove", - "weather", - "trash", - "cable", - "regret", - "buddy", - "beach", - "historian", - "courage", - "sympathy", - "truck", - "tension", - "permit", - "nose", - "bed", - "son", - "person", - "base", - "meat", - "usual", - "air", - "meeting", - "worth", - "game", - "independence", - "physical", - "brief", - "play", - "raise", - "board", - "she", - "key", - "writing", - "pick", - "command", - "party", - "yesterday", - "spring", - "candidate", - "physics", - "university", - "concern", - "development", - "change", - "string", - "target", - "instance", - "room", - "bitter", - "bird", - "football", - "normal", - "split", - "impression", - "wood", - "long", - "meaning", - "stock", - "cap", - "leadership", - "media", - "ambition", - "fishing", - "essay", - "salad", - "repair", - "today", - "designer", - "night", - "bank", - "drawing", - "inevitable", - "phase", - "vast", - "chip", - "anger", - "switch", - "cry", - "twist", - "personality", - "attempt", - "storage", - "being", - "preparation", - "bat", - "selection", - "white", - "technology", - "contract", - "side", - "section", - "station", - "till", - "structure", - "tongue", - "taste", - "truth", - "difficulty", - "group", - "limit", - "main", - "move", - "feeling", - "light", - "example", - "mission", - "might", - "wait", - "wheel", - "shop", - "host", - "classic", - "alternative", - "cause", - "agent", - "consist", - "table", - "airline", - "text", - "pool", - "craft", - "range", - "fuel", - "tool", - "partner", - "load", - "entrance", - "deposit", - "hate", - "article", - "video", - "summer", - "feature", - "extreme", - "mobile", - "hospital", - "flight", - "fall", - "pension", - "piano", - "fail", - "result", - "rub", - "gap", - "system", - "report", - "suck", - "ordinary", - "wind", - "nerve", - "ask", - "shine", - "note", - "line", - "mom", - "perception", - "brother", - "reference", - "bend", - "charge", - "treat", - "trick", - "term", - "homework", - "bake", - "bid", - "status", - "project", - "strategy", - "orange", - "let", - "enthusiasm", - "parent", - "concentrate", - "device", - "travel", - "poetry", - "business", - "society", - "kiss", - "end", - "vegetable", - "employ", - "schedule", - "hour", - "brave", - "focus", - "process", - "movie", - "illegal", - "general", - "coffee", - "ad", - "highway", - "chemistry", - "psychology", - "hire", - "bell", - "conference", - "relief", - "show", - "neat", - "funny", - "weight", - "quality", - "club", - "daughter", - "zone", - "touch", - "tonight", - "shock", - "burn", - "excuse", - "name", - "survey", - "landscape", - "advance", - "satisfaction", - "bread", - "disaster", - "item", - "hat", - "prior", - "shopping", - "visit", - "east", - "photo", - "home", - "idea", - "father", - "comparison", - "cat", - "pipe", - "winner", - "count", - "lake", - "fight", - "prize", - "foundation", - "dog", - "keep", - "ideal", - "fan", - "struggle", - "peak", - "safety", - "solution", - "hell", - "conclusion", - "population", - "strain", - "alarm", - "measurement", - "second", - "train", - "race", - "due", - "insurance", - "boss", - "tree", - "monitor", - "sick", - "course", - "drag", - "appointment", - "slice", - "still", - "care", - "patience", - "rich", - "escape", - "emotion", - "royal", - "female", - "childhood", - "government", - "picture", - "will", - "sock", - "big", - "gate", - "oil", - "cross", - "pin", - "improvement", - "championship", - "silly", - "help", - "sky", - "pitch", - "man", - "diamond", - "most", - "transition", - "work", - "science", - "committee", - "moment", - "fix", - "teaching", - "dig", - "specialist", - "complex", - "guide", - "people", - "dead", - "voice", - "original", - "break", - "topic", - "data", - "degree", - "reading", - "recording", - "bunch", - "reach", - "judgment", - "lie", - "regular", - "set", - "painting", - "mode", - "list", - "player", - "bear", - "north", - "wonder", - "carpet", - "heavy", - "officer", - "negative", - "clock", - "unique", - "baby", - "pain", - "assumption", - "disk", - "iron", - "bill", - "drawer", - "look", - "double", - "mistake", - "finish", - "future", - "brilliant", - "contact", - "math", - "rice", - "leave", - "restaurant", - "discount", - "sex", - "virus", - "bit", - "trust", - "event", - "wear", - "juice", - "failure", - "bug", - "context", - "mud", - "whole", - "wrap", - "intention", - "draft", - "pressure", - "cake", - "dark", - "explanation", - "space", - "angle", - "word", - "efficiency", - "management", - "habit", - "star", - "chance", - "finding", - "transportation", - "stand", - "criticism", - "flow", - "door", - "injury", - "insect", - "surprise", - "apartment", -] # pylint: disable=line-too-long - -# ISO 639-1 codes to language names. -LANGUAGE_CODES = { - "en": "English", - "es": "Spanish", - "pt": "Portuguese", - "ar": "Arabic", - "hi": "Hindi", - "fr": "French", - "ru": "Russian", - "de": "German", - "ja": "Japanese", - "it": "Italian", - "bn": "Bengali", - "uk": "Ukrainian", - "th": "Thai", - "ur": "Urdu", - "ta": "Tamil", - "te": "Telugu", - "bg": "Bulgarian", - "ko": "Korean", - "pl": "Polish", - "he": "Hebrew", - "fa": "Persian", - "vi": "Vietnamese", - "ne": "Nepali", - "sw": "Swahili", - "kn": "Kannada", - "mr": "Marathi", - "gu": "Gujarati", - "pa": "Punjabi", - "ml": "Malayalam", - "fi": "Finnish", -} - -_ALPHABETS = "([A-Za-z])" -_PREFIXES = "(Mr|St|Mrs|Ms|Dr)[.]" -_SUFFIXES = "(Inc|Ltd|Jr|Sr|Co)" -_STARTERS = ( - r"(Mr|Mrs|Ms|Dr|Prof|Capt|Cpt|Lt|He\s|She\s|It\s|They\s|Their\s|Our\s|We\s|But\s|However\s|That\s|This\s|Wherever)" -) -_ACRONYMS = "([A-Z][.][A-Z][.](?:[A-Z][.])?)" -_WEBSITES = "[.](com|net|org|io|gov|edu|me)" -_DIGITS = "([0-9])" -_MULTIPLE_DOTS = r"\.{2,}" - - -def split_into_sentences(text): - """Split the text into sentences. - - Args: - text: A string that consists of more than or equal to one sentences. - - Returns: - A list of strings where each string is a sentence. - """ - text = " " + text + " " - text = text.replace("\n", " ") - text = re.sub(_PREFIXES, "\\1", text) - text = re.sub(_WEBSITES, "\\1", text) - text = re.sub(_DIGITS + "[.]" + _DIGITS, "\\1\\2", text) - text = re.sub( - _MULTIPLE_DOTS, - lambda match: "" * len(match.group(0)) + "", - text, - ) - if "Ph.D" in text: - text = text.replace("Ph.D.", "PhD") - text = re.sub(r"\s" + _ALPHABETS + "[.] ", " \\1 ", text) - text = re.sub(_ACRONYMS + " " + _STARTERS, "\\1 \\2", text) - text = re.sub( - _ALPHABETS + "[.]" + _ALPHABETS + "[.]" + _ALPHABETS + "[.]", - "\\1\\2\\3", - text, - ) - text = re.sub(_ALPHABETS + "[.]" + _ALPHABETS + "[.]", "\\1\\2", text) - text = re.sub(" " + _SUFFIXES + "[.] " + _STARTERS, " \\1 \\2", text) - text = re.sub(" " + _SUFFIXES + "[.]", " \\1", text) - text = re.sub(" " + _ALPHABETS + "[.]", " \\1", text) - if "”" in text: - text = text.replace(".”", "”.") - if '"' in text: - text = text.replace('."', '".') - if "!" in text: - text = text.replace('!"', '"!') - if "?" in text: - text = text.replace('?"', '"?') - text = text.replace(".", ".") - text = text.replace("?", "?") - text = text.replace("!", "!") - text = text.replace("", ".") - sentences = text.split("") - sentences = [s.strip() for s in sentences] - if sentences and not sentences[-1]: - sentences = sentences[:-1] - return sentences - - -def count_words(text): - """Counts the number of words (split on white spaces). - - Returns: - int: The number of words in the text - """ - tokenizer = nltk.tokenize.RegexpTokenizer(r"\w+") - tokens = tokenizer.tokenize(text) - num_words = len(tokens) - return num_words - - -@functools.lru_cache(maxsize=None) -def _get_sentence_tokenizer(): - return nltk.data.load("nltk:tokenizers/punkt/english.pickle") - - -def count_stopwords(text): - """Counts the number of stopwords.""" - nltk.download("stopwords") - stopwords = nltk.corpus.stopwords.words("english") - tokenizer = nltk.tokenize.RegexpTokenizer(r"\w+") - tokens = tokenizer.tokenize(text) - num_stopwords = len([t for t in tokens if t.lower() in stopwords]) - return num_stopwords - - -def count_sentences(text): - """Count the number of sentences (split on ``). - - Returns: - int: The number of sentences in the text - """ - tokenizer = _get_sentence_tokenizer() - tokenized_sentences = tokenizer.tokenize(text) - return len(tokenized_sentences) - - -def generate_keywords(num_keywords): - """Randomly generates a few keywords. - - Returns: - list[str]: List of randomly selected keywords - """ - return random.sample(WORD_LIST, k=num_keywords) diff --git a/src/lighteval/tasks/tasks/ifeval/main.py b/src/lighteval/tasks/tasks/ifeval/main.py deleted file mode 100644 index 5fce23699..000000000 --- a/src/lighteval/tasks/tasks/ifeval/main.py +++ /dev/null @@ -1,239 +0,0 @@ -""" -name: -IFEval - -dataset: -google/IFEval - -abstract: -Very specific task where there are no precise outputs but instead we test if the -format obeys rules. - -languages: -english - -tags: -instruction-following - -paper: -https://arxiv.org/abs/2311.07911 - -starred: -true -""" - -import numpy as np -from inspect_ai.dataset import Sample -from inspect_ai.scorer import Score, Target, accuracy, scorer, stderr -from inspect_ai.solver import TaskState, generate - -import lighteval.tasks.tasks.ifeval.instructions_registry as instructions_registry -from lighteval.metrics.metrics_sample import SampleLevelComputation -from lighteval.metrics.utils.metric_utils import ( - SampleLevelMetricGrouping, -) -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc, SamplingMethod -from lighteval.utils.imports import requires - - -# Very specific task where there are no precise outputs but instead we test if the format obeys rules -@requires("langdetect") -def ifeval_prompt(line, task_name: str = ""): - return Doc( - task_name=task_name, - query=line["prompt"], - choices=[""], - gold_index=0, - instruction="", - specific={"instructions_id_list": line["instruction_id_list"], "kwargs": line["kwargs"]}, - ) - - -submetric_names = [ - "prompt_level_strict_acc", - "inst_level_strict_acc", - "prompt_level_loose_acc", - "inst_level_loose_acc", -] - -REASONING_TAG_PAIRS = [ - ("", ""), -] - - -def _preprocess_response(response: str) -> str: - all_responses = [] - r = response.split("\n") - response_remove_first = "\n".join(r[1:]).strip() - response_remove_last = "\n".join(r[:-1]).strip() - response_remove_both = "\n".join(r[1:-1]).strip() - revised_response = response.replace("*", "") - revised_response_remove_first = response_remove_first.replace("*", "") - revised_response_remove_last = response_remove_last.replace("*", "") - revised_response_remove_both = response_remove_both.replace("*", "") - all_responses = [ - response, - revised_response, - response_remove_first, - response_remove_last, - response_remove_both, - revised_response_remove_first, - revised_response_remove_last, - revised_response_remove_both, - ] - return all_responses - - -class IFEvalMetrics(SampleLevelComputation): - def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> dict: - response = model_response.final_text[0] - - # Strict instructions - instruction_list = doc.specific["instructions_id_list"] - all_kwargs = doc.specific["kwargs"] - prompt = doc.query - - # Loose instructions - all_responses = _preprocess_response(response) - - is_following_list_strict = [] - is_following_list_loose = [] - - for index, instruction_id in enumerate(instruction_list): - instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id] - instruction = instruction_cls(instruction_id) - - # Remove None values from kwargs to avoid unexpected keyword argument errors in build_description method. - task_kwargs = {k: v for k, v in all_kwargs[index].items() if v} - instruction.build_description(**task_kwargs) - args = instruction.get_instruction_args() - if args and "prompt" in args: - instruction.build_description(prompt=prompt) - - # Strict - if response.strip() and instruction.check_following(response): - is_following_list_strict.append(True) - else: - is_following_list_strict.append(False) - - # Loose - is_following = False - for r in all_responses: - if r.strip() and instruction.check_following(r): - is_following = True - break - - is_following_list_loose.append(is_following) - - return { - "prompt_level_strict_acc": int(all(is_following_list_strict)), - "inst_level_strict_acc": is_following_list_strict, - "prompt_level_loose_acc": int(all(is_following_list_loose)), - "inst_level_loose_acc": is_following_list_loose, - } - - -@requires("langdetect") -def agg_inst_level_acc(items): - flat_items = [item for sublist in items for item in sublist] - inst_level_acc = sum(flat_items) / len(flat_items) - return inst_level_acc - - -ifeval_metrics = SampleLevelMetricGrouping( - metric_name=submetric_names, - higher_is_better=dict.fromkeys(submetric_names, True), - category=SamplingMethod.GENERATIVE, - sample_level_fn=IFEvalMetrics(), - corpus_level_fn={ - "prompt_level_strict_acc": np.mean, - "inst_level_strict_acc": agg_inst_level_acc, - "prompt_level_loose_acc": np.mean, - "inst_level_loose_acc": agg_inst_level_acc, - }, -) - - -@requires("langdetect") -def record_to_sample(record): - metadata = {"instruction_id_list": record["instruction_id_list"], "kwargs": record["kwargs"]} - return Sample( - input=record["prompt"], - metadata=metadata, - ) - - -@scorer( - metrics={ - "prompt_level_strict_acc": [accuracy(), stderr()], - "prompt_level_loose_acc": [accuracy(), stderr()], - } -) -def ifeval_scorer(): - async def score(state: TaskState, target: Target): - response = state.output.completion - # Strict instructions - instruction_list = state.metadata["instruction_id_list"] - all_kwargs = state.metadata["kwargs"] - prompt = state.input - # Loose instructions - all_responses = _preprocess_response(response) - - is_following_list_strict = [] - is_following_list_loose = [] - for index, instruction_id in enumerate(instruction_list): - instruction_cls = instructions_registry.INSTRUCTION_DICT[instruction_id] - instruction = instruction_cls(instruction_id) - # Remove None values from kwargs to avoid unexpected keyword argument errors in build_description method. - task_kwargs = {k: v for k, v in all_kwargs[index].items() if v} - instruction.build_description(**task_kwargs) - args = instruction.get_instruction_args() - if args and "prompt" in args: - instruction.build_description(prompt=prompt) - # Strict - if response.strip() and instruction.check_following(response): - is_following_list_strict.append(True) - else: - is_following_list_strict.append(False) - # Loose - is_following = False - for r in all_responses: - if r.strip() and instruction.check_following(r): - is_following = True - break - - is_following_list_loose.append(is_following) - return Score( - value={ - "prompt_level_strict_acc": int(all(is_following_list_strict)), - "prompt_level_loose_acc": int(all(is_following_list_loose)), - }, - explanation=str(instruction_list), - ) - - return score - - -# We create the task config -ifeval = LightevalTaskConfig( - name="ifeval", - prompt_function=ifeval_prompt, - hf_repo="google/IFEval", - hf_subset="default", - metrics=[ifeval_metrics], - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split="train", - few_shots_select="random_sampling", - generation_size=1280, - stop_sequence=[], # no stop sequence, will use eot token - version="0.1", - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=ifeval_scorer(), -) - - -TASKS_TABLE = [ifeval] diff --git a/src/lighteval/tasks/tasks/imdb.py b/src/lighteval/tasks/tasks/imdb.py deleted file mode 100644 index d6d0d9715..000000000 --- a/src/lighteval/tasks/tasks/imdb.py +++ /dev/null @@ -1,86 +0,0 @@ -""" -name: -Imdb - -dataset: -lighteval/IMDB_helm - -abstract: -The IMDB benchmark for sentiment analysis in movie review, from: -Learning Word Vectors for Sentiment Analysis - -languages: -english - -tags: -classification - -paper: -https://aclanthology.org/P11-1015/ -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def imdb_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"Passage: {line['input']}\nSentiment: ", - choices=["Positive", "Negative"], - gold_index=["Positive", "Negative"].index(line["reference"]), - ) - - -def imdb_contrastset_prompt(line, task_name: str = None): - if line["contrast_input"] is None or line["contrast_references"] is None: - return imdb(line) - - return Doc( - task_name=task_name, - query=f"Passage: {line['contrast_inputs']}\nSentiment: ", - choices=["Positive", "Negative"], - gold_index=["Positive", "Negative"].index(line["contrast_references"]), - ) - - -imdb = LightevalTaskConfig( - name="imdb", - prompt_function=imdb_prompt, - hf_repo="lighteval/IMDB_helm", - hf_subset="default", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - - -imdb_contrastset = LightevalTaskConfig( - name="imdb:contrastset", - prompt_function=imdb_contrastset_prompt, - hf_repo="lighteval/IMDB_helm", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - imdb, - imdb_contrastset, -] diff --git a/src/lighteval/tasks/tasks/jeopardy.py b/src/lighteval/tasks/tasks/jeopardy.py deleted file mode 100644 index d31960f9e..000000000 --- a/src/lighteval/tasks/tasks/jeopardy.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -name: -Jeopardy - -dataset: -openaccess-ai-collective/jeopardy - -abstract: -Jeopardy is a dataset of questions and answers from the Jeopardy game show. - -languages: -english - -tags: -knowledge, qa - -paper: -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -jeopardy = LightevalTaskConfig( - name="jeopardy", - prompt_function=get_qa_prompt_function( - Language.ENGLISH, - lambda line: { - "question": line["question"], - "choices": [line["answer"]], - }, - ), - hf_repo="openaccess-ai-collective/jeopardy", - hf_subset="default", - evaluation_splits=("train",), - few_shots_split="train", - generation_size=250, - stop_sequence=["\n", "Question:", "question:"], - metrics=[Metrics.exact_match], - version=1, -) - -TASKS_TABLE = [ - jeopardy, -] diff --git a/src/lighteval/tasks/tasks/lambada.py b/src/lighteval/tasks/tasks/lambada.py deleted file mode 100644 index a039f3d33..000000000 --- a/src/lighteval/tasks/tasks/lambada.py +++ /dev/null @@ -1,86 +0,0 @@ -""" -name: -Lambada - -dataset: -cimec/lambada - -abstract: -LAMBADA is a benchmark for testing language models’ ability to understand broad -narrative context. Each passage requires predicting its final word—easy for -humans given the full passage but impossible from just the last sentence. -Success demands long-range discourse comprehension. - -languages: -english - -tags: -language-modeling - -paper: -https://arxiv.org/abs/1606.06031 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -lambada_cloze_query_suffix = " ____. ->" - - -def lambada_cloze_prompt(line, task_name: str = None): - query, choice = line["text"].rsplit(" ", 1) - return Doc( - task_name=task_name, - query=f"{query}{lambada_cloze_query_suffix}", - gold_index=0, - choices=[f" {choice}"], - ) - - -def lambada_prompt(line, task_name: str = None): - query, choice = line["text"].rsplit(" ", 1) - return Doc( - task_name=task_name, - query=query, - gold_index=0, - choices=[f" {choice}"], - ) - - -lambada_standard = LightevalTaskConfig( - name="lambada:standard", - prompt_function=lambada_prompt, - hf_repo="cimec/lambada", - hf_subset="plain_text", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=10, - metrics=[Metrics.target_perplexity], - stop_sequence=["\n"], - version=0, -) - - -lambada_standard_cloze = LightevalTaskConfig( - name="lambada:standard_cloze", - prompt_function=lambada_cloze_prompt, - hf_repo="cimec/lambada", - hf_subset="plain_text", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=10, - metrics=[Metrics.target_perplexity], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - lambada_standard, - lambada_standard_cloze, -] diff --git a/src/lighteval/tasks/tasks/lcb/codegen_metrics.py b/src/lighteval/tasks/tasks/lcb/codegen_metrics.py deleted file mode 100644 index e2617ed44..000000000 --- a/src/lighteval/tasks/tasks/lcb/codegen_metrics.py +++ /dev/null @@ -1,664 +0,0 @@ -""" -name: -Codegen Metrics - -dataset: - -abstract: - -languages: - -tags: - -paper: -""" - -import ast -import base64 -import faulthandler -import json -import multiprocessing -import os -import pickle -import platform - -# to run the solution files we're using a timing based approach -import signal -import sys -import time -import zlib -from concurrent.futures import ProcessPoolExecutor, as_completed -from decimal import Decimal -from enum import Enum -from io import StringIO -from types import ModuleType -from typing import Callable, Optional - -# used for testing the code that reads from input -from unittest.mock import mock_open, patch - -import numpy as np -from tqdm import tqdm - - -try: - sys.set_int_max_str_digits(50000) -except AttributeError: - print("You likely won't be able to run codegen metrics on your system.") - -os.environ["TOKENIZERS_PARALLELISM"] = "false" - -import_string = "from string import *\nfrom re import *\nfrom datetime import *\nfrom collections import *\nfrom heapq import *\nfrom bisect import *\nfrom copy import *\nfrom math import *\nfrom random import *\nfrom statistics import *\nfrom itertools import *\nfrom functools import *\nfrom operator import *\nfrom io import *\nfrom sys import *\nfrom json import *\nfrom builtins import *\nfrom typing import *\nimport string\nimport re\nimport datetime\nimport collections\nimport heapq\nimport bisect\nimport copy\nimport math\nimport random\nimport statistics\nimport itertools\nimport functools\nimport operator\nimport io\nimport sys\nimport json\nsys.setrecursionlimit(50000)\n" - - -class CODE_TYPE(Enum): - call_based = 0 - standard_input = 1 - - -# stuff for setting up signal timer -class TimeoutException(Exception): - pass - - -def timeout_handler(signum, frame): - print("timeout occurred: alarm went off") - raise TimeoutException - - -# used to capture stdout as a list -# from https://stackoverflow.com/a/16571630/6416660 -# alternative use redirect_stdout() from contextlib -class Capturing(list): - def __enter__(self): - self._stdout = sys.stdout - sys.stdout = self._stringio = StringIO() - # Make closing the StringIO a no-op - self._stringio.close = lambda x: 1 - return self - - def __exit__(self, *args): - self.append(self._stringio.getvalue()) - del self._stringio # free up some memory - sys.stdout = self._stdout - - -def clean_if_name(code: str) -> str: - try: - astree = ast.parse(code) - last_block = astree.body[-1] - if isinstance(last_block, ast.If): - condition = last_block.test - if ast.unparse(condition).strip() == "__name__ == '__main__'": - code = ( - ast.unparse(astree.body[:-1]) + "\n" + ast.unparse(last_block.body) # type: ignore - ) - except Exception: - pass - - return code - - -def make_function(code: str) -> str: - try: - import_stmts = [] - all_other_stmts = [] - astree = ast.parse(code) - for stmt in astree.body: - if isinstance(stmt, (ast.Import, ast.ImportFrom)): - import_stmts.append(stmt) - else: - all_other_stmts.append(stmt) - - function_ast = ast.FunctionDef( - name="wrapped_function", - args=ast.arguments(posonlyargs=[], args=[], kwonlyargs=[], kw_defaults=[], defaults=[]), - body=all_other_stmts, - decorator_list=[], - lineno=-1, - ) - main_code = ( - import_string - + "\n" - + ast.unparse(import_stmts) # type: ignore - + "\n" - + ast.unparse(function_ast) # type: ignore - ) - return main_code - except Exception: - return code - - -def call_method(method: Callable, inputs: list | str): - if isinstance(inputs, list): - inputs = "\n".join(inputs) - - inputs_line_iterator = iter(inputs.split("\n")) - - @patch("builtins.open", mock_open(read_data=inputs)) - @patch("sys.stdin", StringIO(inputs)) - @patch("sys.stdin.readline", lambda *args: next(inputs_line_iterator)) - @patch("sys.stdin.readlines", lambda *args: inputs.split("\n")) - @patch("sys.stdin.read", lambda *args: inputs) - def _inner_call_method(_method): - try: - return _method() - except SystemExit: - pass - finally: - pass - - return _inner_call_method(method) - - -def get_function(compiled_sol, fn_name: str) -> str: # type: ignore - try: - assert hasattr(compiled_sol, fn_name) - return getattr(compiled_sol, fn_name) - except Exception: - return - - -def compile_code(code: str, timeout: int) -> ModuleType: - signal.alarm(timeout) - try: - tmp_sol = ModuleType("tmp_sol", "") - exec(code, tmp_sol.__dict__) - if "class Solution" in code: - # leetcode wraps solutions in `Solution` - # this is a hack to check if it is leetcode solution or not - # currently livecodebench only supports LeetCode but - # else condition allows future extensibility to other platforms - compiled_sol = tmp_sol.Solution() - else: - # do nothing in the other case since function is accessible - compiled_sol = tmp_sol - - assert compiled_sol is not None - finally: - signal.alarm(0) - - return compiled_sol - - -def convert_line_to_decimals(line: str) -> tuple[bool, list[Decimal]]: - try: - decimal_line = [Decimal(elem) for elem in line.split()] - except Exception: - return False, [] - return True, decimal_line - - -def get_stripped_lines(val: str) -> list[str]: - # you don't want empty lines to add empty list after splitlines! - val = val.strip() - - return [val_line.strip() for val_line in val.split("\n")] - - -def grade_call_based(code: str, all_inputs: list, all_outputs: list, fn_name: str, timeout: int) -> list[int | bool]: - # call-based clean up logic - # need to wrap in try-catch logic after to catch the correct errors, but for now this is fine. - code = import_string + "\n\n" + code - compiled_sol = compile_code(code, timeout) - if compiled_sol is None: - # The code could not be compiled as the text was maybe malformed, return [-4] error code. - return [-4] - - method = get_function(compiled_sol, fn_name) - if method is None: - # The function to evaluate could not be extracted from the code, return [-4] error code. - return [-4] - - all_inputs = [[json.loads(line) for line in inputs.split("\n")] for inputs in all_inputs] - - all_outputs = [json.loads(output) for output in all_outputs] - - total_execution = 0 - all_results = [] - for idx, (gt_inp, gt_out) in enumerate(zip(all_inputs, all_outputs)): - signal.alarm(timeout) - faulthandler.enable() - try: - # can lock here so time is useful - start = time.time() - prediction = method(*gt_inp) - total_execution += time.time() - start - signal.alarm(0) - - # don't penalize model if it produces tuples instead of lists - # ground truth sequences are not tuples - if isinstance(prediction, tuple): - prediction = list(prediction) - - tmp_result = prediction == gt_out - - all_results.append(tmp_result) - - if not tmp_result: - return all_results - - except Exception as e: - signal.alarm(0) - if "timeoutexception" in repr(e).lower(): - all_results.append(-3) - return all_results - - else: - all_results.append(-4) - return all_results - - finally: - signal.alarm(0) - faulthandler.disable() - - return all_results - - -def grade_stdio( # noqa: C901 - code: str, - all_inputs: list, - all_outputs: list, - timeout: int, -) -> list[int | bool]: - # runtime doesn't interact well with __name__ == '__main__' - code = clean_if_name(code) - - # we wrap the given code inside another function - code = make_function(code) - - compiled_sol = compile_code(code, timeout) - if compiled_sol is None: - # The code could not be compiled as the text was maybe malformed, return [-4] error code. - return [-4] - - method = get_function(compiled_sol, "wrapped_function") - - if method is None: - # The function to evaluate could not be extracted from the code, return [-4] error code. - return [-4] - - all_results = [] - total_execution_time = 0 - for idx, (gt_inp, gt_out) in enumerate(zip(all_inputs, all_outputs)): - signal.alarm(timeout) - faulthandler.enable() - - signal.alarm(timeout) - with Capturing() as captured_output: - try: - start = time.time() - call_method(method, gt_inp) - total_execution_time += time.time() - start - # reset the alarm - signal.alarm(0) - except Exception as e: - signal.alarm(0) - if "timeoutexception" in repr(e).lower(): - all_results.append(-3) - return all_results - - else: - all_results.append(-4) - return all_results - - finally: - signal.alarm(0) - faulthandler.disable() - - prediction = captured_output[0] - - stripped_prediction_lines = get_stripped_lines(prediction) - stripped_gt_out_lines = get_stripped_lines(gt_out) - - if len(stripped_prediction_lines) != len(stripped_gt_out_lines): - all_results.append(-2) - return all_results - - for output_line_idx, ( - stripped_prediction_line, - stripped_gt_out_line, - ) in enumerate(zip(stripped_prediction_lines, stripped_gt_out_lines)): - # CASE 1: exact match - if stripped_prediction_line == stripped_gt_out_line: - continue - - # CASE 2: element-wise comparison - # if there are floating elements - # use `decimal` library for good floating point comparison - # otherwise gotcha: np.isclose(50000000000000000, 50000000000000001) = True - # note that we should always be able to convert to decimals - - success, decimal_prediction_line = convert_line_to_decimals(stripped_prediction_line) - if not success: - all_results.append(-2) - return all_results - - success, decimal_gtout_line = convert_line_to_decimals(stripped_gt_out_line) - if not success: - all_results.append(-2) - return all_results - - if decimal_prediction_line == decimal_gtout_line: - continue - - all_results.append(-2) - return all_results - all_results.append(True) - - return all_results - - -def run_test(sample: dict[str, str], test=None, timeout: int = 6) -> list[int | bool]: - """If test(generated_code) is not None it'll try to run the code. - otherwise it'll just return an input and output pair. - The return codes are all for errors. A True value means the function worked, and a False - value means the function did not work. - """ - signal.signal(signal.SIGALRM, timeout_handler) - - # Disable functionalities that can make destructive changes to the test. - # max memory is set to 4GB - reliability_guard() - - try: - in_outs = json.loads(sample["input_output"]) - except ValueError: - in_outs = None - - if in_outs: - if in_outs.get("fn_name") is None: - which_type = CODE_TYPE.standard_input # Standard input - method_name = None - - else: - which_type = CODE_TYPE.call_based # Call-based - method_name = in_outs["fn_name"] - - if test is None: - # assert False, "should not happen: test code is none" - return in_outs - - if which_type == CODE_TYPE.call_based: - signal.alarm(timeout) - try: - return grade_call_based( - code=test, - all_inputs=in_outs["inputs"], - all_outputs=in_outs["outputs"], - fn_name=method_name, - timeout=timeout, - ) - - except Exception: - return [-4] - - finally: - signal.alarm(0) - - elif which_type == CODE_TYPE.standard_input: - signal.alarm(timeout) - try: - return grade_stdio( - code=test, - all_inputs=in_outs["inputs"], - all_outputs=in_outs["outputs"], - timeout=timeout, - ) - - except Exception: - return [-4] - - finally: - signal.alarm(0) - - return [-4] - - -def reliability_guard(maximum_memory_bytes: Optional[int] = None): - """This disables various destructive functions and prevents the generated code - from interfering with the test (e.g. fork bomb, killing other processes, - removing filesystem files, etc.) - - Warning: - This function is NOT a security sandbox. Untrusted code, including, model- - generated code, should not be blindly executed outside of one. See the - Codex paper for more information about OpenAI's code sandbox, and proceed - with caution. - """ - if maximum_memory_bytes is not None: - import resource - - resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes)) - resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes)) - if not platform.uname().system == "Darwin": - resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes)) - - faulthandler.disable() - - import builtins - - # builtins.exit = None - builtins.quit = None - - import os - - os.environ["OMP_NUM_THREADS"] = "1" - - os.kill = None - os.system = None - os.putenv = None - os.remove = None - os.removedirs = None - os.rmdir = None - os.fchdir = None - os.setuid = None - os.fork = None - os.forkpty = None - os.killpg = None - os.rename = None - os.renames = None - os.truncate = None - os.replace = None - os.unlink = None - os.fchmod = None - os.fchown = None - os.chmod = None - os.chown = None - os.chroot = None - os.fchdir = None - os.lchflags = None - os.lchmod = None - os.lchown = None - os.getcwd = None - os.chdir = None - - import shutil - - shutil.rmtree = None - shutil.move = None - shutil.chown = None - - import subprocess - - subprocess.Popen = None # type: ignore - - __builtins__["help"] = None - - import sys - - sys.modules["ipdb"] = None - sys.modules["joblib"] = None - sys.modules["resource"] = None - sys.modules["psutil"] = None - sys.modules["tkinter"] = None - - -def check_correctness(sample, generation, timeout: int) -> list[int | bool]: - """Check correctness of code generation with a global timeout. - The global timeout is to catch some extreme/rare cases not handled by the timeouts - inside `run_test`. - """ - - def _temp_run(sample, generation, result): - result.append(run_test(sample, test=generation, timeout=timeout)) - - manager = multiprocessing.Manager() - result = manager.list() - p = multiprocessing.Process(target=_temp_run, args=(sample, generation, result)) - p.start() - p.join(timeout=(timeout + 1) * len(json.loads(sample["input_output"])["inputs"]) + 5) - if p.is_alive(): - p.kill() - if not result: - in_outs = json.loads(sample["input_output"]) - # consider that all tests failed - result = [[-1 for i in range(len(in_outs["inputs"]))]] - - return result[0] - - -def evaluate_generations_by_problem(args: list[list]) -> list: - problem_generations: list[str] = args[0] - sample = args[1] - timeout: int = args[2] - - res = [] - for o_idx, o in enumerate(problem_generations): - curr_res = [-2] - try: - curr_res = check_correctness(sample, o, timeout=timeout) - fixed = [] - for e in curr_res: - if isinstance(e, np.ndarray): - e = e.item(0) - if isinstance(e, np.bool_): - e = bool(e) - fixed.append(e) - curr_res = fixed - - except Exception as e: - # The compilation failed, keep going - pass - - finally: - assert isinstance(curr_res, list), curr_res - res.append(curr_res) - - return res - - -def evaluate_generations( - samples_list: list, - generations_list: list[list[str]], - num_process_evaluate: int = 16, - timeout=6, -) -> dict[int, list[int | bool]]: - """We take the list of code generations and try to compile them - and the run their corresponding unit tests which are retrieved from the APPS dataset. - - Args: - samples_list (list): List of sample problems from the APPS dataset. - generations_list (list[list[str]]): List of code generations for each sample. - num_process_evaluate (int): Number of processes to use for evaluation. - timeout (int): Timeout for each evaluation in seconds. - - Returns: - dict[int, list[int | bool]]: Dictionary of results, key is the problem index, value is a list of results for each generation - [-2] = compile error, [-1] = runtime error [False] = failed test case [True] = passed test case - """ - # generations are code generations in the same order of the dataset - - inputs = [ - [(generations_list[index], samples_list[index], timeout), index] for index in range(len(generations_list)) - ] - - with tqdm(total=len(inputs)) as pbar: - with ProcessPoolExecutor(max_workers=num_process_evaluate) as executor: - futures = {executor.submit(evaluate_generations_by_problem, arg): index for arg, index in inputs} - - results = {} - for future in as_completed(futures): - index = futures[future] - results[index] = future.result() - pbar.update(1) - - assert len(results) == len(inputs), f"results = {len(results)} inputs = {len(inputs)} {results=}" - - return results - - -def codegen_metrics( - samples, - generations, - k_list=[1, 5], - num_process_evaluate=16, - timeout=6, -): - results = evaluate_generations( - samples, - generations, - num_process_evaluate=num_process_evaluate, - timeout=timeout, - ) - metrics = compute_metrics_from_results(results, k_list=k_list) - - return metrics, results - - -def estimate_pass_at_k(num_samples: np.ndarray, num_correct: np.ndarray, k: int) -> np.ndarray: - """Estimates pass@k of each problem and returns them in an array.""" - - def estimator(n: int, c: int, k: int) -> float: - """Calculates 1 - comb(n - c, k) / comb(n, k).""" - if n - c < k: - return 1.0 - return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1)) - - import itertools - - if isinstance(num_samples, int): - num_samples_it = itertools.repeat(num_samples, len(num_correct)) - else: - assert len(num_samples) == len(num_correct) - num_samples_it = iter(num_samples) - - return np.array([estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)]) - - -def compute_metrics_from_results(results, k_list: list[int] = [1, 5]) -> dict[str, np.ndarray]: - total = [] - correct = [] - task_ids = [] - for task_id, res in results.items(): - all_correct = [] - for generation in res: - gen = np.array(generation) - all_correct.append(np.all(gen > 0)) - task_ids.append(task_id) - total.append(len(all_correct)) - correct.append(sum(all_correct)) - total = np.array(total) - correct = np.array(correct) - detail_pass_at_k = { - f"pass@{k}": estimate_pass_at_k(total, correct, k).tolist() for k in k_list if (total >= k).all() - } - pass_at_k = {f"pass@{k}": estimate_pass_at_k(total, correct, k).mean() for k in k_list if (total >= k).all()} - detail_metrics = {k: dict(zip(task_ids, v)) for k, v in detail_pass_at_k.items()} - pass_at_k["detail"] = detail_metrics - return pass_at_k - - -def translate_private_test_cases(encoded_data: str) -> dict[str, str]: - decoded_data = base64.b64decode(encoded_data) - decompressed_data = zlib.decompress(decoded_data) - original_data = pickle.loads(decompressed_data) - return json.loads(original_data) - - -def extract_code(model_output: str) -> str: - outputlines = model_output.split("\n") - indexlines = [i for i, line in enumerate(outputlines) if "```" in line] - if len(indexlines) < 2: - return "" - return "\n".join(outputlines[indexlines[-2] + 1 : indexlines[-1]]) diff --git a/src/lighteval/tasks/tasks/lcb/main.py b/src/lighteval/tasks/tasks/lcb/main.py deleted file mode 100644 index a3e227288..000000000 --- a/src/lighteval/tasks/tasks/lcb/main.py +++ /dev/null @@ -1,165 +0,0 @@ -""" -name: -Live Code Bench - -dataset: -lighteval/code_generation_lite - -abstract: -LiveCodeBench collects problems from periodic contests on LeetCode, AtCoder, and -Codeforces platforms and uses them for constructing a holistic benchmark for -evaluating Code LLMs across variety of code-related scenarios continuously over -time. - -languages: -english - -tags: -code-generation - -paper: -https://livecodebench.github.io/ - -starred: -true -""" - -import json -from typing import Any - -import numpy as np -from aenum import extend_enum - -from lighteval.metrics.metrics import Metrics, SampleLevelMetric -from lighteval.metrics.metrics_sample import SampleLevelComputation -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.lighteval_task import Doc, LightevalTaskConfig -from lighteval.tasks.requests import SamplingMethod -from lighteval.tasks.tasks.lcb.codegen_metrics import ( - codegen_metrics, - extract_code, - translate_private_test_cases, -) - - -def prepare_prompt(line: dict[str, Any]) -> str: - query = "You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests.\n\n" - query += f"Question: {line['question_content']}\n\n" - if starter_code := line.get("starter_code", None): - query += "You will use the following starter code to write the solution to the problem and enclose your code within delimiters.\n" - query += f"```python\n{starter_code}\n```\n\n" - else: - query += "Read the inputs from stdin solve the problem and write the answer to stdout (do not directly test on the sample inputs). Enclose your code within delimiters as follows. Ensure that when the python program runs, it reads the inputs, runs the algorithm and writes output to STDOUT.\n" - query += "```python\n# YOUR CODE HERE\n```\n\n" - return query - - -def lcb_codegeneration_prompt_fn(line, task_name: str = "lcb:codegeneration") -> Doc: - # For the prompt we need a more general function that can be used tweaked like in: - # https://github.com/LiveCodeBench/LiveCodeBench/blob/main/lcb_runner/prompts/code_generation.py - query = prepare_prompt(line) - # List of dicts of the form: [{"input": "6\nabc\nacb\nbac\nbca\ncab\ncba\n", "output": "YES\nYES\nYES\nNO\nNO\nYES\n", "testtype": "stdin"}] - public_test_cases = json.loads(line["public_test_cases"]) - private_test_cases = translate_private_test_cases(line["private_test_cases"]) - inputs = [test["input"] for test in public_test_cases + private_test_cases] - outputs = [test["output"] for test in public_test_cases + private_test_cases] - return Doc( - task_name=task_name, - query=query, - choices=[""], - gold_index=0, - specific={ - "inputs": inputs, - "outputs": outputs, - "fn_name": json.loads(line["metadata"]).get("func_name", None), - }, - ) - - -class CodegenMetric(SampleLevelComputation): - def compute(self, model_response: ModelResponse, doc: Doc, **kwargs) -> dict: - """Estimates the Pass@1 metric for the code generation task. - Extract the code from each prediction, Runs it for each sample and generations, - and computes the Pass@1 over the outputs. - """ - assert doc.specific is not None, "Doc specific field is required for codegen_metric" - - predictions = model_response.final_text - # Extract generated code snippets - generated_code_snippets = [[extract_code(pred) for pred in predictions]] # noqa: F841 - evaluation_sample = { # noqa: F841 - "inputs": doc.specific["inputs"], - "outputs": doc.specific["outputs"], - "fn_name": doc.specific["fn_name"], - } - # This is a list of lists because - evaluation_sample = [{"input_output": json.dumps(evaluation_sample)}] - - metrics, _ = codegen_metrics( - evaluation_sample, - generated_code_snippets, - k_list=[1], # Only run for Pass@1 - num_process_evaluate=8, - ) - return metrics["pass@1"] - - -lcb_codegen_metric = SampleLevelMetric( - metric_name="codegen_pass@1:16", # This is the way of informing the number of generations currently - category=SamplingMethod.GENERATIVE, - higher_is_better=True, - sample_level_fn=CodegenMetric(), - corpus_level_fn=np.mean, - batched_compute=False, -) - - -extend_enum(Metrics, "lcb_codegen_metric", lcb_codegen_metric) - -configs = [ - "release_v1", - "release_v2", - "release_v3", - "release_v4", - "release_v5", - "release_v6", - "release_latest", - "v1", - "v2", - "v3", - "v4", - "v5", - "v6", - "v1_v2", - "v1_v3", - "v1_v4", - "v1_v5", - "v2_v3", - "v2_v4", - "v2_v5", - "v3_v4", - "v3_v5", - "v4_v5", -] - -tasks = [] - -for subset in configs: - # To keep the base subset as the default, the others are named "lcb:codegeneration_v4", "lcb:codegeneration_v5"... etc - name = "lcb:codegeneration" if subset == "v4_v5" else f"lcb:codegeneration_{subset}" - task = LightevalTaskConfig( - name=name, - prompt_function=lcb_codegeneration_prompt_fn, - hf_repo="lighteval/code_generation_lite", - hf_subset=subset, # https://github.com/LiveCodeBench/LiveCodeBench/tree/main?tab=readme-ov-file#dataset-versions - hf_avail_splits=["test"], - evaluation_splits=["test"], - generation_size=32768, - metrics=[Metrics.lcb_codegen_metric], - stop_sequence=[], # no stop sequence, will use EOS token - version=0, - ) - tasks.append(task) - - -TASKS_TABLE = tasks diff --git a/src/lighteval/tasks/tasks/legal_summarization.py b/src/lighteval/tasks/tasks/legal_summarization.py deleted file mode 100644 index ea27796ba..000000000 --- a/src/lighteval/tasks/tasks/legal_summarization.py +++ /dev/null @@ -1,119 +0,0 @@ -""" -name: -Legal Summarization - -dataset: -lighteval/legal_summarization - -abstract: -LegalSummarization is a dataset for legal summarization. - -languages: -english - -tags: -legal, summarization - -paper: -https://arxiv.org/abs/2210.13448 -https://arxiv.org/abs/2210.13448 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def legal_summarization_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"###\nArticle:{line['text']}\n\nSummarize the above article in 3 sentences.\n", - choices=[str(line["summary"])], - gold_index=0, - specific={"text": line["text"]}, - ) - - -def multilexsum_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"###\nArticle: {line['article']}\n\nSummarize the above article in 2 sentences.\n", - gold_index=0, - choices=[line["summary"]], - specific={"text": line["article"]}, - ) - - -legal_summarization_billsum = LightevalTaskConfig( - name="legal_summarization:billsum", - prompt_function=legal_summarization_prompt, - hf_repo="lighteval/legal_summarization", - hf_subset="BillSum", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=1024, - metrics=[ - Metrics.rouge1, - Metrics.rouge2, - Metrics.rougeL, - Metrics.faithfulness, - Metrics.extractiveness, - Metrics.bert_score, - ], - stop_sequence=["\n"], - version=0, -) - - -legal_summarization_eurlexsum = LightevalTaskConfig( - name="legal_summarization:eurlexsum", - prompt_function=legal_summarization_prompt, - hf_repo="lighteval/legal_summarization", - hf_subset="EurLexSum", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=2048, - metrics=[ - Metrics.rouge1, - Metrics.rouge2, - Metrics.rougeL, - Metrics.faithfulness, - Metrics.extractiveness, - Metrics.bert_score, - ], - stop_sequence=["\n"], - version=0, -) - - -legal_summarization_multilexsum = LightevalTaskConfig( - name="legal_summarization:multilexsum", - prompt_function=multilexsum_prompt, - hf_repo="lighteval/legal_summarization", - hf_subset="MultiLexSum", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=256, - metrics=[ - Metrics.rouge1, - Metrics.rouge2, - Metrics.rougeL, - Metrics.faithfulness, - Metrics.extractiveness, - Metrics.bert_score, - ], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - legal_summarization_billsum, - legal_summarization_eurlexsum, - legal_summarization_multilexsum, -] diff --git a/src/lighteval/tasks/tasks/legalsupport.py b/src/lighteval/tasks/tasks/legalsupport.py deleted file mode 100644 index 4893755d4..000000000 --- a/src/lighteval/tasks/tasks/legalsupport.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -name: -Legalsupport - -dataset: -lighteval/LegalSupport - -abstract: -Measures fine-grained legal reasoning through reverse entailment. - -languages: -english - -tags: -legal - -paper: -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def legalsupport_prompt(line, task_name: str = None): - query = f"Which statement best supports the passage?\nPassage: {line['context']}\n" - query += "".join( - [ - f"{key}. {choice}\n" - for key, choice in zip( - ["a", "b"], [line["citation_a"]["parenthetical"], line["citation_b"]["parenthetical"]] - ) - ] - ) - query += "Answer:" - - return Doc( - task_name=task_name, - query=query, - choices=["a", "b"], - gold_index=0 if line["answer_label"] == "citation_a" else 1, - ) - - -legalsupport = LightevalTaskConfig( - name="legalsupport", - prompt_function=legalsupport_prompt, - hf_repo="lighteval/LegalSupport", - hf_subset="default", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - legalsupport, -] diff --git a/src/lighteval/tasks/tasks/lexglue.py b/src/lighteval/tasks/tasks/lexglue.py deleted file mode 100644 index cd02ddf72..000000000 --- a/src/lighteval/tasks/tasks/lexglue.py +++ /dev/null @@ -1,184 +0,0 @@ -""" -name: -Lexglue - -dataset: -lighteval/lexglue - -abstract: -LexGLUE: A Benchmark Dataset for Legal Language Understanding in English - -languages: -english - -tags: -classification, legal - -paper: -https://arxiv.org/abs/2110.00976 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def lex_glue(line, instruction, task_name: str = None): - return Doc( - task_name=task_name, - query=f"{instruction}\nPassage: {line['input']}\nAnswer: ", - choices=line["references"], - gold_index=[line["references"].index(item) for item in line["gold"]], - instruction=instruction + "\n", - ) - - -def lex_glue_ecthr_a_prompt(line, task_name: str = None): - instruction = "In this task, you are given the facts from a case heard at the European Court of Human Rights (ECtHR). Predict the articles of the ECtHR that were violated (if any)." - return lex_glue(line, instruction, task_name) - - -def lex_glue_ecthr_b_prompt(line, task_name: str = None): - instruction = "In this task, you are given the facts from a case heard at the European Court of Human Rights (ECtHR). Predict the articles of ECtHR that were allegedly violated (considered by the court)." - return lex_glue(line, instruction, task_name) - - -def lex_glue_scotus_prompt(line, task_name: str = None): - instruction = "In this task, you are given a case heard at the Supreme Court of the United States (SCOTUS). Predict the relevant issue area." - return lex_glue(line, instruction, task_name) - - -def lex_glue_eurlex_prompt(line, task_name: str = None): - instruction = "In this task, you are given an EU law document published in the EUR-Lex portal. Predict the relevant EuroVoc concepts." - return lex_glue(line, instruction, task_name) - - -def lex_glue_ledgar_prompt(line, task_name: str = None): - instruction = "In this task, you are given a contract provision \nfrom contracts obtained from US Securities and Exchange Commission (SEC) filings. Predict the main topic." - return lex_glue(line, instruction, task_name) - - -def lex_glue_unfair_tos_prompt(line, task_name: str = None): - instruction = "In this task, you are given a sentence \nfrom a Terms of Service (ToS) document from on-line platforms. Predict the types of unfair contractual terms" - return lex_glue(line, instruction, task_name) - - -def lex_glue_case_hold_prompt(line, task_name: str = None): - instruction = "In this task, you are given an excerpt from a court decision, \ncontaining a reference to a particular case, while the holding statement is masked out. Predict the index of the holding statement fitting in the context at from a selection of five choices." - return lex_glue(line, instruction, task_name) - - -lexglue_case_hold = LightevalTaskConfig( - name="lexglue:case_hold", - prompt_function=lex_glue_case_hold_prompt, - hf_repo="lighteval/lexglue", - hf_subset="case_hold", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lexglue_ecthr_a = LightevalTaskConfig( - name="lexglue:ecthr_a", - prompt_function=lex_glue_ecthr_a_prompt, - hf_repo="lighteval/lexglue", - hf_subset="ecthr_a", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lexglue_ecthr_b = LightevalTaskConfig( - name="lexglue:ecthr_b", - prompt_function=lex_glue_ecthr_b_prompt, - hf_repo="lighteval/lexglue", - hf_subset="ecthr_b", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lexglue_eurlex = LightevalTaskConfig( - name="lexglue:eurlex", - prompt_function=lex_glue_eurlex_prompt, - hf_repo="lighteval/lexglue", - hf_subset="eurlex", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lexglue_ledgar = LightevalTaskConfig( - name="lexglue:ledgar", - prompt_function=lex_glue_ledgar_prompt, - hf_repo="lighteval/lexglue", - hf_subset="ledgar", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lexglue_scotus = LightevalTaskConfig( - name="lexglue:scotus", - prompt_function=lex_glue_scotus_prompt, - hf_repo="lighteval/lexglue", - hf_subset="scotus", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lexglue_unfair_tos = LightevalTaskConfig( - name="lexglue:unfair_tos", - prompt_function=lex_glue_unfair_tos_prompt, - hf_repo="lighteval/lexglue", - hf_subset="unfair_tos", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - lexglue_case_hold, - lexglue_ecthr_a, - lexglue_ecthr_b, - lexglue_eurlex, - lexglue_ledgar, - lexglue_scotus, - lexglue_unfair_tos, -] diff --git a/src/lighteval/tasks/tasks/lextreme.py b/src/lighteval/tasks/tasks/lextreme.py deleted file mode 100644 index 6d564c9bb..000000000 --- a/src/lighteval/tasks/tasks/lextreme.py +++ /dev/null @@ -1,485 +0,0 @@ -""" -name: -Lextreme - -dataset: -lighteval/lextreme - -abstract: -LEXTREME: A Multi-Lingual and Multi-Task Benchmark for the Legal Domain - -languages: -bulgarian, czech, danish, german, greek, english, spanish, estonian, finnish, french, ga, croatian, hungarian, italian, lithuanian, latvian, mt, dutch, polish, portuguese, romanian, slovak, slovenian, swedish - -tags: -classification, legal - -paper: -https://arxiv.org/abs/2301.13126 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def lextreme_prompt(line, instruction, task_name: str = None): - return Doc( - task_name=task_name, - query=f"{instruction}\nPassage: {line['input']}\nAnswer: ", - choices=line["references"], - gold_index=[line["references"].index(item) for item in line["gold"]], - instruction=instruction + "\n", - ) - - -def lextreme_brazilian_court_decisions_judgment_prompt(line, task_name: str = None): - instruction = ( - "In this task, you are given the case description " - "from a decision heard at the State Supreme Court of Alagoas (Brazil). " - "Predict the judgment of the case " - "(no: The appeal was denied, " - "partial: For partially favourable decisions, " - "yes: For fully favourable decisions)" - ) - return lextreme_prompt(line, instruction, task_name) - - -def lextreme_brazilian_court_decisions_unanimity_prompt(line, task_name: str = None): - instruction = ( - "In this task, you are given the case description " - "from a decision heard at the State Supreme Court of Alagoas (Brazil). " - "Predict the unanimity of the case (unanimity, not-unanimity, not_determined)" - ) - return lextreme_prompt(line, instruction, task_name) - - -def lextreme_german_argument_mining_prompt(line, task_name: str = None): - instruction = ( - "In this task, you are given sentences from German court decisions. " - "Predict the major component of German Urteilsstil " - "(conclusion: Overall result, " - "definition: Abstract legal facts and consequences, " - "subsumption: Determination sentence / Concrete facts, " - "other: Anything else)" - ) - return lextreme_prompt(line, instruction, task_name) - - -def lextreme_greek_legal_code_chapter_prompt(line, task_name: str = None): - instruction = ( - "In this task, you are given a Greek legislative document. " - "Predict the chapter level category of the " - "'Permanent Greek Legislation Code - Raptarchis (Ραπτάρχης)' the document belongs to." - ) - return lextreme_prompt(line, instruction, task_name) - - -def lextreme_greek_legal_code_subject_prompt(line, task_name: str = None): - instruction = ( - "In this task, you are given a Greek legislative document. " - "Predict the subject level category of the " - "'Permanent Greek Legislation Code - Raptarchis (Ραπτάρχης)' the document belongs to." - ) - - return lextreme_prompt(line, instruction, task_name) - - -def lextreme_greek_legal_code_volume_prompt(line, task_name: str = None): - instruction = ( - "In this task, you are given a Greek legislative document. " - "Predict the volume level category of the " - "'Permanent Greek Legislation Code - Raptarchis (Ραπτάρχης)' the document belongs to." - ) - return lextreme_prompt(line, instruction, task_name) - - -def lextreme_swiss_judgment_prediction_prompt(line, task_name: str = None): - instruction = ( - "In this task, you are given the facts description " - "from a decision heard at the Swiss Federal Supreme Court. " - "Predict the judgment of the case (approval or dismissal)" - ) - return lextreme_prompt(line, instruction, task_name) - - -def lextreme_online_terms_of_service_unfairness_levels_prompt(line, task_name: str = None): - instruction = ( - "In this task, you are given a sentence " - "from a Terms of Service (ToS) document. " - "Predict the unfairness level of the sentence (potentially_unfair, clearly_unfair, clearly_fair, untagged)" - ) - return lextreme_prompt(line, instruction, task_name) - - -def lextreme_online_terms_of_service_clause_topics_prompt(line, task_name: str = None): - instruction = ( - "In this task, you are given a sentence " - "from a Terms of Service (ToS) document. " - "Predict the clause topics of the sentence " - "(0: Arbitration, " - "1: Unilateral change, " - "2: Content removal, " - "3: Jurisdiction, " - "4: Choice of law, " - "5: Limitation of liability, " - "6: Unilateral termination, " - "7: Contract by using, " - "8: Privacy included)" - ) - return lextreme_prompt(line, instruction, task_name) - - -def lextreme_covid19_emergency_event_prompt(line, task_name: str = None): - instruction = ( - "In this task, you are given a sentence from a European legislative document. " - "Predict the applicable measurements against COVID-19 " - "(0: State of Emergency, " - "1: Restrictions of fundamental rights and civil liberties, " - "2: Restrictions of daily liberties, " - "3: Closures / lockdown, " - "4: Suspension of international cooperation and commitments, " - "5: Police mobilization, " - "6: Army mobilization, " - "7: Government oversight)" - ) - - return lextreme_prompt(line, instruction, task_name) - - -def lextreme_multi_eurlex_level_1_prompt(line, task_name: str = None): - instruction = ( - "In this task, you are given a document from an EU law. Predict the level 1 concept in the EUROVOC taxonomy." - ) - return lextreme_prompt(line, instruction, task_name) - - -def lextreme_multi_eurlex_level_2_prompt(line, task_name: str = None): - instruction = ( - "In this task, you are given a document from an EU law. Predict the level 2 concept in the EUROVOC taxonomy." - ) - return lextreme_prompt(line, instruction, task_name) - - -def lextreme_multi_eurlex_level_3_prompt(line, task_name: str = None): - instruction = ( - "In this task, you are given a document from an EU law. Predict the level 3 concept in the EUROVOC taxonomy." - ) - return lextreme_prompt(line, instruction, task_name) - - -def lextreme_greek_legal_ner_prompt(line, task_name: str = None): - instruction = "In this task, you are given a Greek legal document. Predict the named entities." - return lextreme_prompt(line, instruction, task_name) - - -def lextreme_legalnero_prompt(line, task_name: str = None): - instruction = "In this task, you are given a legal text. Predict the named entities of legal interest." - return lextreme_prompt(line, instruction, task_name) - - -def lextreme_lener_br_prompt(line, task_name: str = None): - instruction = "In this task, you are given a Brazilian legal text. Predict the named entities." - return lextreme_prompt(line, instruction, task_name) - - -def lextreme_mapa_coarse_prompt(line, task_name: str = None): - instruction = "In this task, you are given a legal text. Predict the coarse-grained labels." - return lextreme_prompt(line, instruction, task_name) - - -def lextreme_mapa_fine_prompt(line, task_name: str = None): - instruction = "In this task, you are given a legal text. Predict the fine-grained labels." - return lextreme_prompt(line, instruction, task_name) - - -lextreme_brazilian_court_decisions_judgment = LightevalTaskConfig( - name="lextreme:brazilian_court_decisions_judgment", - prompt_function=lextreme_brazilian_court_decisions_judgment_prompt, - hf_repo="lighteval/lextreme", - hf_subset="brazilian_court_decisions_judgment", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lextreme_brazilian_court_decisions_unanimity = LightevalTaskConfig( - name="lextreme:brazilian_court_decisions_unanimity", - prompt_function=lextreme_brazilian_court_decisions_unanimity_prompt, - hf_repo="lighteval/lextreme", - hf_subset="brazilian_court_decisions_unanimity", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lextreme_covid19_emergency_event = LightevalTaskConfig( - name="lextreme:covid19_emergency_event", - prompt_function=lextreme_covid19_emergency_event_prompt, - hf_repo="lighteval/lextreme", - hf_subset="covid19_emergency_event", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=10, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lextreme_german_argument_mining = LightevalTaskConfig( - name="lextreme:german_argument_mining", - prompt_function=lextreme_german_argument_mining_prompt, - hf_repo="lighteval/lextreme", - hf_subset="german_argument_mining", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lextreme_greek_legal_code_chapter = LightevalTaskConfig( - name="lextreme:greek_legal_code_chapter", - prompt_function=lextreme_greek_legal_code_chapter_prompt, - hf_repo="lighteval/lextreme", - hf_subset="greek_legal_code_chapter", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lextreme_greek_legal_code_subject = LightevalTaskConfig( - name="lextreme:greek_legal_code_subject", - prompt_function=lextreme_greek_legal_code_subject_prompt, - hf_repo="lighteval/lextreme", - hf_subset="greek_legal_code_subject", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lextreme_greek_legal_code_volume = LightevalTaskConfig( - name="lextreme:greek_legal_code_volume", - prompt_function=lextreme_greek_legal_code_volume_prompt, - hf_repo="lighteval/lextreme", - hf_subset="greek_legal_code_volume", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lextreme_greek_legal_ner = LightevalTaskConfig( - name="lextreme:greek_legal_ner", - prompt_function=lextreme_greek_legal_ner_prompt, - hf_repo="lighteval/lextreme", - hf_subset="greek_legal_ner", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=430, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lextreme_legalnero = LightevalTaskConfig( - name="lextreme:legalnero", - prompt_function=lextreme_legalnero_prompt, - hf_repo="lighteval/lextreme", - hf_subset="legalnero", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=788, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lextreme_lener_br = LightevalTaskConfig( - name="lextreme:lener_br", - prompt_function=lextreme_lener_br_prompt, - hf_repo="lighteval/lextreme", - hf_subset="lener_br", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=338, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lextreme_mapa_coarse = LightevalTaskConfig( - name="lextreme:mapa_coarse", - prompt_function=lextreme_mapa_coarse_prompt, - hf_repo="lighteval/lextreme", - hf_subset="mapa_coarse", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=274, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lextreme_mapa_fine = LightevalTaskConfig( - name="lextreme:mapa_fine", - prompt_function=lextreme_mapa_fine_prompt, - hf_repo="lighteval/lextreme", - hf_subset="mapa_fine", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=274, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lextreme_multi_eurlex_level_1 = LightevalTaskConfig( - name="lextreme:multi_eurlex_level_1", - prompt_function=lextreme_multi_eurlex_level_1_prompt, - hf_repo="lighteval/lextreme", - hf_subset="multi_eurlex_level_1", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=10, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lextreme_multi_eurlex_level_2 = LightevalTaskConfig( - name="lextreme:multi_eurlex_level_2", - prompt_function=lextreme_multi_eurlex_level_2_prompt, - hf_repo="lighteval/lextreme", - hf_subset="multi_eurlex_level_2", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=10, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lextreme_multi_eurlex_level_3 = LightevalTaskConfig( - name="lextreme:multi_eurlex_level_3", - prompt_function=lextreme_multi_eurlex_level_3_prompt, - hf_repo="lighteval/lextreme", - hf_subset="multi_eurlex_level_3", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=10, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lextreme_online_terms_of_service_clause_topics = LightevalTaskConfig( - name="lextreme:online_terms_of_service_clause_topics", - prompt_function=lextreme_online_terms_of_service_clause_topics_prompt, - hf_repo="lighteval/lextreme", - hf_subset="online_terms_of_service_clause_topics", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=10, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lextreme_online_terms_of_service_unfairness_levels = LightevalTaskConfig( - name="lextreme:online_terms_of_service_unfairness_levels", - prompt_function=lextreme_online_terms_of_service_unfairness_levels_prompt, - hf_repo="lighteval/lextreme", - hf_subset="online_terms_of_service_unfairness_levels", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=10, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lextreme_swiss_judgment_prediction = LightevalTaskConfig( - name="lextreme:swiss_judgment_prediction", - prompt_function=lextreme_swiss_judgment_prediction_prompt, - hf_repo="lighteval/lextreme", - hf_subset="swiss_judgment_prediction", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - lextreme_brazilian_court_decisions_judgment, - lextreme_brazilian_court_decisions_unanimity, - lextreme_covid19_emergency_event, - lextreme_german_argument_mining, - lextreme_greek_legal_code_chapter, - lextreme_greek_legal_code_subject, - lextreme_greek_legal_code_volume, - lextreme_greek_legal_ner, - lextreme_legalnero, - lextreme_lener_br, - lextreme_mapa_coarse, - lextreme_mapa_fine, - lextreme_multi_eurlex_level_1, - lextreme_multi_eurlex_level_2, - lextreme_multi_eurlex_level_3, - lextreme_online_terms_of_service_clause_topics, - lextreme_online_terms_of_service_unfairness_levels, - lextreme_swiss_judgment_prediction, -] diff --git a/src/lighteval/tasks/tasks/logiqa.py b/src/lighteval/tasks/tasks/logiqa.py deleted file mode 100644 index 03461f3ac..000000000 --- a/src/lighteval/tasks/tasks/logiqa.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -name: -Logiqa - -dataset: -lighteval/logiqa_harness - -abstract: -LogiQA is a machine reading comprehension dataset focused on testing logical -reasoning abilities. It contains 8,678 expert-written multiple-choice questions -covering various types of deductive reasoning. While humans perform strongly, -state-of-the-art models lag far behind, making LogiQA a benchmark for advancing -logical reasoning in NLP systems. - -languages: -english - -tags: -qa - -paper: -https://arxiv.org/abs/2007.08124 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def logiqa_prompt(line, task_name: str = None): - query = f"Passage: {line['context']}\nQuestion: {line['question']}\nChoices:\n" - query += "".join([f"{key}. {choice}\n" for key, choice in zip(["A", "B", "C", "D"], line["options"])]) - query += "Answer:" - - return Doc( - task_name=task_name, - query=query, - choices=[f" {c}" for c in line["options"]], - gold_index=["a", "b", "c", "d"].index(line["label"]), - ) - - -logiqa = LightevalTaskConfig( - name="logiqa", - prompt_function=logiqa_prompt, - hf_repo="lighteval/logiqa_harness", - hf_subset="logiqa", - hf_avail_splits=["train", "validation", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - logiqa, -] diff --git a/src/lighteval/tasks/tasks/long_horizon_execution.py b/src/lighteval/tasks/tasks/long_horizon_execution.py deleted file mode 100644 index c3e449991..000000000 --- a/src/lighteval/tasks/tasks/long_horizon_execution.py +++ /dev/null @@ -1,188 +0,0 @@ -""" -name: -Long Horizon Execution - -dataset: -arvindh75/Long-Horizon-Execution - -abstract: -Evaluation benchmark for long-context execution capabilities of language models. -Tests a model's ability to maintain state and perform cumulative operations over -long sequences of inputs. Supports both single-turn (all inputs at once) and -multi-turn (inputs provided incrementally) evaluation modes. -The task requires models to: -1. Maintain a dictionary mapping keys to values -2. Process a sequence of keys -3. Calculate cumulative sums after each key or group of keys -4. Handle varying context sizes and turn complexities -Single-turn evaluation (Section 3.3): Model outputs only the final cumulative sum -after processing all keys, allowing any aggregation strategy. - -Multi-turn evaluation: Model processes keys in batches of K per turn, maintaining -conversation history and outputting cumulative sums incrementally. Evaluates -fractional accuracy (correct turns / total turns). - -languages: -english - -tags: -long-context, state-tracking, arithmetic, execution - -paper: -https://arxiv.org/abs/2509.09677 - -starred: -true -""" - -import functools -import itertools -import re - -from inspect_ai.dataset import Sample -from inspect_ai.model import ChatMessageUser -from inspect_ai.scorer import Score, Target, accuracy, mean, scorer -from inspect_ai.solver import Generate, TaskState, solver - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig - - -PROMPT_TEMPLATE_MULTI_FOLLOWUP = """ -I will now provide you with the next {k} keys to process: - -{keys_str} -""".strip() - -PROMPT_TEMPLATE_MULTI_START = """ -I will provide you with a dictionary and then give you the first {k} keys to process. -Your task is to keep a running total (starting from 0) by adding the values associated with the keys I provide. -In each turn, I'll provide {k} keys (comma-separated). -Respond with the current running sum, enclosed in tags. - -Dictionary to maintain: -{dict_str} - -Ready to start! - -{keys_str} -""".strip() - - -def record_to_sample(record, k=1, max_turns=5): - input_keys, input_values = record["input"], record["values"] - - dictionary = dict(zip(input_keys, input_values)) - dictionary_str = str(dictionary) - - keys_per_turn = [input_keys[i : i + k] for i in range(0, len(input_keys), k)][:max_turns] - values_per_turn = [input_values[i : i + k] for i in range(0, len(input_values), k)][:max_turns] - - targets_per_turn = list(itertools.accumulate(sum(values) for values in values_per_turn)) - - initial_prompt = PROMPT_TEMPLATE_MULTI_START.format(dict_str=dictionary_str, keys_str=str(keys_per_turn[0]), k=k) - - metadata = { - "keys_per_turn": keys_per_turn, - "values_per_turn": values_per_turn, - "targets_per_turn": targets_per_turn, - "k": k, - "max_turns": max_turns, - } - - return Sample( - input=initial_prompt, - target=str(targets_per_turn[-1]), # last turn cumulative sum - metadata=metadata, - ) - - -@solver -def solver(): - async def solve(state: TaskState, generate: Generate): - keys_per_turn = state.metadata["keys_per_turn"] - - all_turn_outputs = [] - - # Process first turn (already in messages as initial prompt) - state = await generate(state) - all_turn_outputs.append(state.output.completion) - - # Process remaining turns - for keys in keys_per_turn[1:]: - keys_str = ", ".join(keys) - followup_prompt = PROMPT_TEMPLATE_MULTI_FOLLOWUP.format(keys_str=keys_str, k=state.metadata["k"]) - state.messages.append(ChatMessageUser(content=followup_prompt)) - state = await generate(state) - all_turn_outputs.append(state.output.completion) - - state.metadata["all_turn_outputs"] = all_turn_outputs - - return state - - return solve - - -@scorer(metrics={"horizon": [mean()], "turn_accuracy": [mean()], "all_correct": [accuracy()]}) -def scorer(): - answer_pattern = re.compile(r"(.*?)", re.DOTALL) - - async def score(state: TaskState, target: Target): - all_turn_outputs = state.metadata.get("all_turn_outputs", []) - targets_per_turn = state.metadata.get("targets_per_turn", []) - - parsed_outputs = [] - - for turn_output in all_turn_outputs: - match = answer_pattern.search(turn_output) - if match: - content = match.group(1).strip() - try: - parsed_value = int(content) - parsed_outputs.append(parsed_value) - except ValueError: - parsed_outputs.append(None) - - turn_results = [] - for turn_output, target in zip(parsed_outputs, targets_per_turn): - is_correct = (turn_output is not None) and (turn_output == target) - turn_results.append({"output": turn_output, "target": target, "correct": is_correct}) - - turn_accuracy = sum(result["correct"] for result in turn_results) / len(turn_results) - - # Horizon: first turn (0-indexed) where the model was not correct anymore - # If all turns are correct, horizon is len(turn_results) (number of turns completed) - horizon = len(turn_results) - for turn_idx, result in enumerate(turn_results): - if not result["correct"]: - horizon = turn_idx - break - - return Score( - value={ - "turn_accuracy": turn_accuracy, - "horizon": horizon, - "all_correct": all(result["correct"] for result in turn_results), - }, - answer=str(turn_results), - explanation=state.output.completion, - ) - - return score - - -long_horizon_execution_10 = LightevalTaskConfig( - name="long_horizon_execution", - prompt_function=lambda line, task_name: line, - sample_fields=functools.partial(record_to_sample, k=10, max_turns=30), - solver=[solver()], - scorer=[scorer()], - hf_repo="arvindh75/Long-Horizon-Execution", - hf_subset="default", - evaluation_splits=("test",), - metrics=[Metrics.exact_match], -) - -TASKS_TABLE = [ - long_horizon_execution_10, -] diff --git a/src/lighteval/tasks/tasks/lsat_qa.py b/src/lighteval/tasks/tasks/lsat_qa.py deleted file mode 100644 index 713a1d3b2..000000000 --- a/src/lighteval/tasks/tasks/lsat_qa.py +++ /dev/null @@ -1,121 +0,0 @@ -""" -name: -Lsat Qa - -dataset: -lighteval/lsat_qa - -abstract: -Questions from law school admission tests. - -languages: -english - -tags: -legal, qa - -paper: -""" - -from string import ascii_uppercase - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def lsat_qa_prompt(line, task_name: str = None): - query = f"The following are multiple choice questions (with answers).\nPassage: {line['passage']}\nQuestion: {line['question']}\n" - query += "".join([f"{key}. {choice}\n" for key, choice in zip(ascii_uppercase, line["references"])]) - query += "Answer:" - return Doc( - task_name=task_name, - query=query, - choices=list(ascii_uppercase)[: len(line["references"])], - gold_index=line["gold_index"], - instruction="The following are multiple choice questions (with answers).\n", - ) - - -lsat_qa = LightevalTaskConfig( - name="lsat_qa", - prompt_function=lsat_qa_prompt, - hf_repo="lighteval/lsat_qa", - hf_subset="all", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lsat_qa_assignment = LightevalTaskConfig( - name="lsat_qa:assignment", - prompt_function=lsat_qa_prompt, - hf_repo="lighteval/lsat_qa", - hf_subset="assignment", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lsat_qa_grouping = LightevalTaskConfig( - name="lsat_qa:grouping", - prompt_function=lsat_qa_prompt, - hf_repo="lighteval/lsat_qa", - hf_subset="grouping", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lsat_qa_miscellaneous = LightevalTaskConfig( - name="lsat_qa:miscellaneous", - prompt_function=lsat_qa_prompt, - hf_repo="lighteval/lsat_qa", - hf_subset="miscellaneous", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -lsat_qa_ordering = LightevalTaskConfig( - name="lsat_qa:ordering", - prompt_function=lsat_qa_prompt, - hf_repo="lighteval/lsat_qa", - hf_subset="ordering", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - lsat_qa, - lsat_qa_assignment, - lsat_qa_grouping, - lsat_qa_miscellaneous, - lsat_qa_ordering, -] diff --git a/src/lighteval/tasks/tasks/math.py b/src/lighteval/tasks/tasks/math.py deleted file mode 100644 index 74909d9c8..000000000 --- a/src/lighteval/tasks/tasks/math.py +++ /dev/null @@ -1,211 +0,0 @@ -""" -name: -Math - -dataset: -DigitalLearningGmbH/MATH-lighteval - -abstract: - -languages: -english - -tags: -math, reasoning - -paper: -https://arxiv.org/abs/2305.20050 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.metrics.normalizations import math_normalizer -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def math_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"Question: {line['problem']}\nAnswer:", - choices=[f" {line['solution']}"], - gold_index=0, - ) - - -math_algebra = LightevalTaskConfig( - name="math:algebra", - prompt_function=math_prompt, - hf_repo="DigitalLearningGmbH/MATH-lighteval", - hf_subset="algebra", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=2048, - metrics=[ - Metrics.maj_at_n( - sample_params={ - "n": 4, - "strip_strings": True, - "normalize_pred": math_normalizer, - "normalize_gold": math_normalizer, - } - ), - ], - stop_sequence=["\n"], - version=1, -) - -math_counting_and_probability = LightevalTaskConfig( - name="math:counting_and_probability", - prompt_function=math_prompt, - hf_repo="DigitalLearningGmbH/MATH-lighteval", - hf_subset="counting_and_probability", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=2048, - metrics=[ - Metrics.maj_at_n( - sample_params={ - "n": 4, - "strip_strings": True, - "normalize_pred": math_normalizer, - "normalize_gold": math_normalizer, - } - ), - ], - stop_sequence=["\n"], - version=1, -) - -math_geometry = LightevalTaskConfig( - name="math:geometry", - prompt_function=math_prompt, - hf_repo="DigitalLearningGmbH/MATH-lighteval", - hf_subset="geometry", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=2048, - metrics=[ - Metrics.maj_at_n( - sample_params={ - "n": 4, - "strip_strings": True, - "normalize_pred": math_normalizer, - "normalize_gold": math_normalizer, - } - ), - ], - stop_sequence=["\n"], - version=1, -) - -math_intermediate_algebra = LightevalTaskConfig( - name="math:intermediate_algebra", - prompt_function=math_prompt, - hf_repo="DigitalLearningGmbH/MATH-lighteval", - hf_subset="intermediate_algebra", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=2048, - metrics=[ - Metrics.maj_at_n( - sample_params={ - "n": 4, - "strip_strings": True, - "normalize_pred": math_normalizer, - "normalize_gold": math_normalizer, - } - ), - ], - stop_sequence=["\n"], - version=1, -) - -math_number_theory = LightevalTaskConfig( - name="math:number_theory", - prompt_function=math_prompt, - hf_repo="DigitalLearningGmbH/MATH-lighteval", - hf_subset="number_theory", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=2048, - metrics=[ - Metrics.maj_at_n( - sample_params={ - "n": 4, - "strip_strings": True, - "normalize_pred": math_normalizer, - "normalize_gold": math_normalizer, - } - ), - ], - stop_sequence=["\n"], - version=1, -) - -math_prealgebra = LightevalTaskConfig( - name="math:prealgebra", - prompt_function=math_prompt, - hf_repo="DigitalLearningGmbH/MATH-lighteval", - hf_subset="prealgebra", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=2048, - metrics=[ - Metrics.maj_at_n( - sample_params={ - "n": 4, - "strip_strings": True, - "normalize_pred": math_normalizer, - "normalize_gold": math_normalizer, - } - ), - ], - stop_sequence=["\n"], - version=1, -) - -math_precalculus = LightevalTaskConfig( - name="math:precalculus", - prompt_function=math_prompt, - hf_repo="DigitalLearningGmbH/MATH-lighteval", - hf_subset="precalculus", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=2048, - metrics=[ - Metrics.maj_at_n( - sample_params={ - "n": 4, - "strip_strings": True, - "normalize_pred": math_normalizer, - "normalize_gold": math_normalizer, - } - ), - ], - stop_sequence=["\n"], - version=1, -) - -TASKS_TABLE = [ - math_algebra, - math_counting_and_probability, - math_geometry, - math_intermediate_algebra, - math_number_theory, - math_prealgebra, - math_precalculus, -] diff --git a/src/lighteval/tasks/tasks/math_500.py b/src/lighteval/tasks/tasks/math_500.py deleted file mode 100644 index 965bcd645..000000000 --- a/src/lighteval/tasks/tasks/math_500.py +++ /dev/null @@ -1,79 +0,0 @@ -""" -name: -Math 500 - -dataset: -HuggingFaceH4/MATH-500 - -abstract: -This dataset contains a subset of 500 problems from the MATH benchmark that -OpenAI created in their Let's Verify Step by Step paper. - -languages: -english - -tags: -math, reasoning - -paper: -https://arxiv.org/abs/2305.20050 - -starred: -true -""" - -from inspect_ai.dataset import Sample -from inspect_ai.scorer import model_graded_fact -from inspect_ai.solver import generate, prompt_template - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -MATH_QUERY_TEMPLATE = """ -Solve the following problem. The final line of your response MUST be of the following format: -"ANSWER: $ANSWER" (without quotes) where $ANSWER is the final answer. Think step by step before answering. - -{prompt} -""".strip() - - -def math_500_prompt(line, task_name: str = None): - query = MATH_QUERY_TEMPLATE.format(prompt=line["problem"]) - return Doc( - task_name=task_name, - query=query, - choices=[f"ANSWER: {line['solution']}"], - gold_index=0, - ) - - -def record_to_sample(record): - query = record["problem"] - target = record["answer"] - return Sample(input=query, target=target) - - -math_500 = LightevalTaskConfig( - name="math_500", - prompt_function=math_500_prompt, - hf_repo="HuggingFaceH4/MATH-500", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=32768, - metrics=[ - Metrics.pass_at_k_math(sample_params={"k": 1, "n": 1}), - ], - version=2, - sample_fields=record_to_sample, - solver=[prompt_template(MATH_QUERY_TEMPLATE), generate(cache=True)], - scorer=model_graded_fact(), -) - -TASKS_TABLE = [ - math_500, -] diff --git a/src/lighteval/tasks/tasks/mathqa.py b/src/lighteval/tasks/tasks/mathqa.py deleted file mode 100644 index 4bf6706aa..000000000 --- a/src/lighteval/tasks/tasks/mathqa.py +++ /dev/null @@ -1,69 +0,0 @@ -""" -name: -Mathqa - -dataset: -allenai/math_qa - -abstract: -large-scale dataset of math word problems. Our dataset is gathered by using a -new representation language to annotate over the AQuA-RAT dataset with -fully-specified operational programs. AQuA-RAT has provided the questions, -options, rationale, and the correct options. - -languages: -english - -tags: -math, qa, reasoning - -paper: -https://arxiv.org/abs/1905.13319 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def mathqa_prompt(line, task_name: str = None): - query = f"Problem: {line['Problem']}\n" - query += "Options:\n" - query += "".join( - [ - f"{key}) {choice}\n" - for key, choice in zip( - ["a", "b", "c", "d", "e"], - [line["option_a"], line["option_b"], line["option_c"], line["option_d"], line["option_e"]], - ) - ] - ) - query += "Answer:" - return Doc( - task_name=task_name, - query=query, - choices=[ - f" {c}" for c in [line["option_a"], line["option_b"], line["option_c"], line["option_d"], line["option_e"]] - ], - gold_index=["a", "b", "c", "d", "e"].index(line["correct"]), - ) - - -mathqa = LightevalTaskConfig( - name="mathqa", - prompt_function=mathqa_prompt, - hf_repo="allenai/math_qa", - hf_subset="default", - hf_avail_splits=["train", "validation", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - mathqa, -] diff --git a/src/lighteval/tasks/tasks/mathvista.py b/src/lighteval/tasks/tasks/mathvista.py deleted file mode 100644 index 1ba759938..000000000 --- a/src/lighteval/tasks/tasks/mathvista.py +++ /dev/null @@ -1,217 +0,0 @@ -""" -name: -MathVista - -dataset: -AI4Math/MathVista - -abstract: -Large Language Models (LLMs) and Large Multimodal Models (LMMs) exhibit impressive problem-solving skills in many tasks and domains, but their ability in mathematical reasoning in visual contexts has not been systematically studied. To bridge this gap, we present MathVista, a benchmark designed to combine challenges from diverse mathematical and visual tasks. It consists of 6,141 examples, derived from 28 existing multimodal datasets involving mathematics and 3 newly created datasets (i.e., IQTest, FunctionQA, and PaperQA). Completing these tasks requires fine-grained, deep visual understanding and compositional reasoning, which all state-of-the-art foundation models find challenging. - -languages: -english - -tags: -math, qa, reasoning - -paper: -https://arxiv.org/pdf/2310.02255 - -starred: -true -""" - -import logging -import re -from io import BytesIO -from pathlib import Path -from string import ascii_uppercase -from typing import Any - -from inspect_ai.dataset import Sample -from inspect_ai.model import ChatMessage, ChatMessageUser, ContentImage, ContentText -from inspect_ai.scorer import ( - CORRECT, - INCORRECT, - AnswerPattern, - Score, - Scorer, - Target, - accuracy, - scorer, - stderr, -) -from inspect_ai.scorer._pattern import match_first -from inspect_ai.solver import ( - Generate, - Solver, - TaskState, - multiple_choice, - solver, - system_message, -) -from PIL import Image - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig - - -# code insperied by: https://github.com/UKGovernmentBEIS/inspect_evals/blob/main/src/inspect_evals/mathvista/mathvista.py - -logger = logging.getLogger(__name__) - -SYSTEM_PROMPT = """ -Answer must come after an "ANSWER:" tag. -""".strip() - - -def is_image_png(image_bytes: bytes) -> bool: - return image_bytes[:8] == b"\x89\x50\x4e\x47\x0d\x0a\x1a\x0a" - - -def is_image_webp(image_bytes: bytes) -> bool: - return image_bytes[:4] == b"RIFF" and image_bytes[8:12] == b"WEBP" - - -def get_message(record: dict[str, Any]) -> Sample: - # extract image - IMAGE_BASE_DIR = Path("./data/mathvista_images") - image = Path(IMAGE_BASE_DIR / record["image"]) - - # images are a mix of jpg, png, and webp but all have a file extension of .jpg - image_bytes = record["decoded_image"]["bytes"] - if is_image_png(image_bytes): - image = image.with_suffix(".png") - elif is_image_webp(image_bytes): - image = image.with_suffix(".webp") - - if not image.exists(): - logger.debug(f"Extracting {image.name}") - # ensure parent - image.parent.mkdir(exist_ok=True, parents=True) - # reduce the image size - img = Image.open(BytesIO(image_bytes)) - img.thumbnail((1024, 1024)) - # save preserving format - img.save(image, format=img.format) - - message: list[ChatMessage] = [ - ChatMessageUser( - content=[ - ContentText(text=record["question"]) - if record["question_type"] == "multi_choice" - else ContentText(text=record["query"]), - ContentImage(image=str(image)), - ] - ) - ] - return message - - -def record_to_sample_multiple_choice(record: dict[str, Any]) -> Sample: - message = get_message(record) - target = ascii_uppercase[record["choices"].index(record["answer"])] - - return Sample( - input=message, - choices=record["choices"], - target=target, - id=record["pid"], - metadata={ - "question_type": record["question_type"], - "answer_type": record["answer_type"], - **record["metadata"], - }, - ) - - -def record_to_sample_freeform(record: dict[str, Any]) -> Sample: - message = get_message(record) - return Sample( - input=message, - target=record["answer"], - id=record["pid"], - metadata={ - "precision": record["precision"], - "question_type": record["question_type"], - "answer_type": record["answer_type"], - **record["metadata"], - }, - ) - - -def record_to_sample(record: dict[str, Any]) -> Sample: - if record["question_type"] == "multi_choice": - return record_to_sample_multiple_choice(record) - else: - return record_to_sample_freeform(record) - - -@scorer(metrics=[accuracy(), stderr()]) -def mathvista_scorer() -> Scorer: - async def score(state: TaskState, target: Target) -> Score: - match = re.search( - AnswerPattern.LETTER if state.metadata["question_type"] == "multi_choice" else AnswerPattern.LINE, - state.output.completion, - re.IGNORECASE, - ) - if match: - # scoring pattern found in response - groups = match.groups() - found_match = match_first(matches=groups, target=target, ignore_case=True) - - if found_match is None and len(groups) == 1: - answer = groups[0] - else: - answer = found_match - - return Score( - value=CORRECT if found_match else INCORRECT, - answer=answer, - explanation=state.output.completion, - ) - - else: - # didn't find the scoring pattern - return Score( - value=INCORRECT, - explanation="Scoring pattern not matched in output: " + f"{state.output.completion}", - ) - - return score - - -@solver -def mathvista_solver() -> Solver: - async def solve(state: TaskState, generate: Generate) -> TaskState: - if state.metadata["question_type"] == "multi_choice": - return await multiple_choice()(state, generate) - else: - state = await system_message(SYSTEM_PROMPT)(state, generate) - return await generate(state) - - return solve - - -mathvista = LightevalTaskConfig( - name="mathvista", - prompt_function=lambda x: x, - hf_repo="AI4Math/MathVista", - hf_subset="default", - hf_filter=lambda x: x.get("question_type") == "multi_choice", - hf_avail_splits=["testmini, test"], - evaluation_splits=["testmini"], - few_shots_split=None, - few_shots_select=None, - generation_size=512, - metrics=[Metrics.expr_gold_metric], - stop_sequence=None, - version=0, - sample_fields=record_to_sample, - solver=[mathvista_solver()], - scorer=mathvista_scorer(), -) - -TASKS_TABLE = [ - mathvista, -] diff --git a/src/lighteval/tasks/tasks/med.py b/src/lighteval/tasks/tasks/med.py deleted file mode 100644 index 77552440e..000000000 --- a/src/lighteval/tasks/tasks/med.py +++ /dev/null @@ -1,125 +0,0 @@ -""" -name: -Med - -dataset: -lighteval/med_mcqa, lighteval/med_paragraph_simplification, bigbio/med_qa - -abstract: -A Large-scale Multi-Subject Multi-Choice Dataset for Medical domain Question Answering - -languages: -english - -tags: -health, medical - -paper: -https://medmcqa.github.io/ -""" - -from string import ascii_uppercase - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def med_mcqa_prompt(line, task_name: str = None): - query = f"Give a letter answer among A, B, C or D.\nQuestion: {line['question']}\n" - query += "".join( - [ - f"{key}. {choice}\n" - for key, choice in zip(ascii_uppercase, [line["opa"], line["opb"], line["opc"], line["opd"]]) - ] - ) - query += "Answer:" - return Doc( - task_name=task_name, - query=query, - choices=list(ascii_uppercase)[:4], - gold_index=line["cop"] - 1, - instruction="Give a letter answer among A, B, C or D.\n", - ) - - -def med_paragraph_simplification_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"###\nArticle:{line['query']}\n\nSummarize the above article in 10 sentences.\n", - gold_index=0, - choices=[line["answer"]], - ) - - -def med_qa_prompt(line, task_name: str = None): - query = f"Give a letter answer among A, B, C or D.\nQuestion: {line['question']}\n" - query += "".join([f"{option['key']}. {option['value']}\n" for option in line["options"]]) - query += "Answer:" - return Doc( - task_name=task_name, - query=query, - choices=[opt["key"] for opt in line["options"]], - gold_index=list(ascii_uppercase).index(line["answer_idx"]), - instruction="Give a letter answer among A, B, C or D.\n", - ) - - -med_mcqa = LightevalTaskConfig( - name="med_mcqa", - prompt_function=med_mcqa_prompt, - hf_repo="lighteval/med_mcqa", - hf_subset="default", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - - -med_paragraph_simplification = LightevalTaskConfig( - name="med_paragraph_simplification", - prompt_function=med_paragraph_simplification_prompt, - hf_repo="lighteval/med_paragraph_simplification", - hf_subset="default", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=512, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - - -med_qa = LightevalTaskConfig( - name="med_qa", - prompt_function=med_qa_prompt, - hf_repo="bigbio/med_qa", - hf_subset="med_qa_en_source", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - med_mcqa, - med_paragraph_simplification, - med_qa, -] diff --git a/src/lighteval/tasks/tasks/med_dialog.py b/src/lighteval/tasks/tasks/med_dialog.py deleted file mode 100644 index 0bc61640c..000000000 --- a/src/lighteval/tasks/tasks/med_dialog.py +++ /dev/null @@ -1,72 +0,0 @@ -""" -name: -Med Dialog - -dataset: -lighteval/med_dialog - -abstract: -A collection of medical dialogue datasets. - -languages: -english - -tags: -dialog, health, medical - -paper: -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def med_dialog_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"###\nArticle:{line['src']}\n\nSummarize the above article in 1 sentence.\n", - gold_index=0, - choices=[line["tgt"]], - ) - - -med_dialog_healthcaremagic = LightevalTaskConfig( - name="med_dialog:healthcaremagic", - prompt_function=med_dialog_prompt, - hf_repo="lighteval/med_dialog", - hf_subset="healthcaremagic", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=128, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - - -med_dialog_icliniq = LightevalTaskConfig( - name="med_dialog:icliniq", - prompt_function=med_dialog_prompt, - hf_repo="lighteval/med_dialog", - hf_subset="icliniq", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=128, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - med_dialog_healthcaremagic, - med_dialog_icliniq, -] diff --git a/src/lighteval/tasks/tasks/mgsm.py b/src/lighteval/tasks/tasks/mgsm.py deleted file mode 100644 index 166235d80..000000000 --- a/src/lighteval/tasks/tasks/mgsm.py +++ /dev/null @@ -1,282 +0,0 @@ -""" -name: -Mgsm - -dataset: -juletxara/mgsm - -abstract: -Multilingual Grade School Math Benchmark (MGSM) is a benchmark of grade-school -math problems. -The same 250 problems from GSM8K are each translated via human annotators in 10 -languages. - -languages: -english, spanish, french, german, russian, chinese, japanese, thai, swahili, bengali, telugu - -tags: -math, multilingual, reasoning - -paper: -https://arxiv.org/abs/2210.03057 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def mgsm_prompt(line, question_key, answer_key, task_name: str = None): - if line["answer"] is not None: - query = f"{line['question']}\n{answer_key}" - gold = f" {line['answer'][len(answer_key) + 1 :]}" - else: - query = f"{question_key} {line['question']}\n{answer_key}" - gold = f" {str(line['answer_number'])}" - return Doc(task_name=task_name, query=query, choices=[gold], gold_index=0) - - -def mgsm_en_prompt(line, task_name: str = None): - question_key = "Question:" - answer_key = "Step-by-Step Answer:" - return mgsm_prompt(line, question_key, answer_key, task_name) - - -def mgsm_es_prompt(line, task_name: str = None): - question_key = "Pregunta:" - answer_key = "Respuesta paso a paso:" - return mgsm_prompt(line, question_key, answer_key, task_name) - - -def mgsm_fr_prompt(line, task_name: str = None): - question_key = "Question:" - answer_key = "Réponse étape par étape :" - return mgsm_prompt(line, question_key, answer_key, task_name) - - -def mgsm_de_prompt(line, task_name: str = None): - question_key = "Frage:" - answer_key = "Schritt-für-Schritt-Antwort:" - return mgsm_prompt(line, question_key, answer_key, task_name) - - -def mgsm_ru_prompt(line, task_name: str = None): - question_key = "Задача:" - answer_key = "Пошаговоерешение:" - return mgsm_prompt(line, question_key, answer_key, task_name) - - -def mgsm_zh_prompt(line, task_name: str = None): - question_key = "问题:" - answer_key = "逐步解答:" - return mgsm_prompt(line, question_key, answer_key, task_name) - - -def mgsm_ja_prompt(line, task_name: str = None): - question_key = "問題:" - answer_key = "ステップごとの答え:" - return mgsm_prompt(line, question_key, answer_key, task_name) - - -def mgsm_th_prompt(line, task_name: str = None): - question_key = "โจทย์:" - answer_key = "คำตอบทีละขั้นตอน:" - return mgsm_prompt(line, question_key, answer_key, task_name) - - -def mgsm_sw_prompt(line, task_name: str = None): - question_key = "Swali:" - answer_key = "Jibu la Hatua kwa Hatua:" - return mgsm_prompt(line, question_key, answer_key, task_name) - - -def mgsm_bn_prompt(line, task_name: str = None): - question_key = "প্রশ্ন:" - answer_key = "ধাপে ধাপে উত্তর:" - return mgsm_prompt(line, question_key, answer_key, task_name) - - -def mgsm_te_prompt(line, task_name: str = None): - question_key = "ప్రశ్న:" - answer_key = "దశలవారీగా సమాధానం:" - return mgsm_prompt(line, question_key, answer_key, task_name) - - -mgsm_en = LightevalTaskConfig( - name="mgsm:en", - prompt_function=mgsm_en_prompt, - hf_repo="juletxara/mgsm", - hf_subset="en", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.exact_match], - stop_sequence=None, - version=0, -) - -mgsm_es = LightevalTaskConfig( - name="mgsm:es", - prompt_function=mgsm_es_prompt, - hf_repo="juletxara/mgsm", - hf_subset="es", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.exact_match], - stop_sequence=None, - version=0, -) - -mgsm_fr = LightevalTaskConfig( - name="mgsm:fr", - prompt_function=mgsm_fr_prompt, - hf_repo="juletxara/mgsm", - hf_subset="fr", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.exact_match], - stop_sequence=None, - version=0, -) - -mgsm_de = LightevalTaskConfig( - name="mgsm:de", - prompt_function=mgsm_de_prompt, - hf_repo="juletxara/mgsm", - hf_subset="de", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.exact_match], - stop_sequence=None, - version=0, -) - -mgsm_ru = LightevalTaskConfig( - name="mgsm:ru", - prompt_function=mgsm_ru_prompt, - hf_repo="juletxara/mgsm", - hf_subset="ru", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.exact_match], - stop_sequence=None, - version=0, -) - -mgsm_zh = LightevalTaskConfig( - name="mgsm:zh", - prompt_function=mgsm_zh_prompt, - hf_repo="juletxara/mgsm", - hf_subset="zh", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.exact_match], - stop_sequence=None, - version=0, -) - -mgsm_ja = LightevalTaskConfig( - name="mgsm:ja", - prompt_function=mgsm_ja_prompt, - hf_repo="juletxara/mgsm", - hf_subset="ja", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.exact_match], - stop_sequence=None, - version=0, -) - -mgsm_th = LightevalTaskConfig( - name="mgsm:th", - prompt_function=mgsm_th_prompt, - hf_repo="juletxara/mgsm", - hf_subset="th", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.exact_match], - stop_sequence=None, - version=0, -) - -mgsm_sw = LightevalTaskConfig( - name="mgsm:sw", - prompt_function=mgsm_sw_prompt, - hf_repo="juletxara/mgsm", - hf_subset="sw", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.exact_match], - stop_sequence=None, - version=0, -) - -mgsm_bn = LightevalTaskConfig( - name="mgsm:bn", - prompt_function=mgsm_bn_prompt, - hf_repo="juletxara/mgsm", - hf_subset="bn", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.exact_match], - stop_sequence=None, - version=0, -) - -mgsm_te = LightevalTaskConfig( - name="mgsm:te", - prompt_function=mgsm_te_prompt, - hf_repo="juletxara/mgsm", - hf_subset="te", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.exact_match], - stop_sequence=None, - version=0, -) - -TASKS_TABLE = [ - mgsm_en, - mgsm_es, - mgsm_fr, - mgsm_de, - mgsm_ru, - mgsm_zh, - mgsm_ja, - mgsm_th, - mgsm_sw, - mgsm_bn, - mgsm_te, -] diff --git a/src/lighteval/tasks/tasks/mix_eval/judge_prompts.py b/src/lighteval/tasks/tasks/mix_eval/judge_prompts.py deleted file mode 100644 index 48850b820..000000000 --- a/src/lighteval/tasks/tasks/mix_eval/judge_prompts.py +++ /dev/null @@ -1,221 +0,0 @@ -from lighteval.tasks.tasks.mix_eval.prompts import parse_options - - -def flow_judge_for_freeform_template(question, options, answer, gold): - return [ - { - "role": "user", - "content": f"""# GOAL -Your job is to evaluate a task carried out by an AI system powered by a large \ -language model. - -You will be provided with the inputs and output of the task, as well as the evaluation criteria \ -and scoring rubric. Your task is to evaluate the output of the AI system based on the evaluation \ -criteria and scoring rubric provided. - -# INPUT -Below are the inputs required for performing the task: - -{question} - - -# OUTPUT -Below is the output of the task: - -{answer} - - -# EVALUATION CRITERIA AND SCORING RUBRIC -Here are the evaluation criteria and the rubric that you need to use for evaluating the task: - -How well the response answers the question, the reference answer is: -{gold} - - - -- Score 1: The response completely fails to answer the question. -- Score 2: The response barely answers the question. -- Score 3: The response partially answers the question. -- Score 4: The response mostly answers the question. -- Score 5: The response completely answers the question. - - -# INSTRUCTIONS FOR THE EVALUATION -1. Understand the task and criteria: Familiarize yourself with the task to be evaluated. \ -Review the evaluation criteria and scoring rubric to understand the different levels of \ -performance and the descriptions for each score. -2. Review the inputs and output: Look at the inputs provided for the task. Examine the output \ -generated from completing the task. -3. Compare output to score descriptions: Compare the output against the criteria and score \ -descriptions in the scoring rubric. For each criterion,decide which description best matches the \ -output. -4. After comparing the output to the score descriptions, pay attention to the small details that \ -might impact the final score that you assign. Sometimes a small difference can dictate the final \ -score. -5. Write verbal feedback justifying your evaluation that includes a detailed rationale, referring \ -to specific aspects of the output and comparing them to the rubric. -6. Assign a final score based on the scoring rubric. - -## FORMAT FOR THE EVALUATION -- Write the verbal feedback inside tags without any additional surrounding text. -- Write the numeric score inside tags, without any additional surrounding text and always \ -after the feedback. - -Please accurately evaluate the task. Strictly adhere to the evaluation criteria and rubric.""", - } - ] - - -def flow_judge_for_multichoice_template(question, options, answer, gold): - return [ - { - "role": "user", - "content": f"""# GOAL -Your job is to evaluate a task carried out by an AI system powered by a large \ -language model. - -You will be provided with the inputs and output of the task, as well as the evaluation criteria \ -and scoring rubric. Your task is to evaluate the output of the AI system based on the evaluation \ -criteria and scoring rubric provided. - -# INPUT -Below are the inputs required for performing the task: - -{question} -options: -{parse_options(options)} - - -# OUTPUT -Below is the output of the task: - -{answer} - - -# EVALUATION CRITERIA AND SCORING RUBRIC -Here are the evaluation criteria and the rubric that you need to use for evaluating the task: - -The correct option for this task is: -{gold} - -Did the model choose the correct option? - - - -- score 0: The model did not choose the correct option. -- score 1: The model chose the correct option. - - -# INSTRUCTIONS FOR THE EVALUATION -1. Understand the task and criteria: Familiarize yourself with the task to be evaluated. \ -Review the evaluation criteria and scoring rubric to understand the different levels of \ -performance and the descriptions for each score. -2. Review the inputs and output: Look at the inputs provided for the task. Examine the output \ -generated from completing the task. -3. Compare output to score descriptions: Compare the output against the criteria and score \ -descriptions in the scoring rubric. For each criterion,decide which description best matches the \ -output. -4. After comparing the output to the score descriptions, pay attention to the small details that \ -might impact the final score that you assign. Sometimes a small difference can dictate the final \ -score. -5. Write verbal feedback justifying your evaluation that includes a detailed rationale, referring \ -to specific aspects of the output and comparing them to the rubric. -6. Assign a final score based on the scoring rubric. - -## FORMAT FOR THE EVALUATION -- Write the verbal feedback inside tags without any additional surrounding text. -- Write the numeric score inside tags, without any additional surrounding text and always \ -after the feedback. - -Please accurately evaluate the task. Strictly adhere to the evaluation criteria and rubric.""", - } - ] - - -# Judge Prompts for Close-ended Free-form Parser############ -# gpt_judge_for_closeended_freeform = lambda question, options, answer, gold: [ -def gpt_judge_for_closeended_freeform(question, options, answer, gold): - return [ - {"role": "system", "content": "In this task, I want you to act as a judge."}, - { - "role": "user", - "content": f"""You will be provided with a question, its golden answer(s), and the model's answer, while the context of the question is not given here. Your task is to judge how correct the model's answer is based on the golden answer(s), without seeing the context of the question, and then give a correctness score. The correctness score should be one of the below numbers: 0.0 (totally wrong), 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, or 1.0 (totally right). Your should first briefly give your reasoning process regarding how the model's answer conforms to or contradicts the golden answer(s), and then give the correctness score. The correctness score must strictly follow this format: \"[[score]]\", e.g., \"The correctness score: [[0.5]]\". Below are some examples. - -Example 1: -Question: Sandy bought 1 million Safe Moon tokens. She has 4 siblings. She wants to keep half of them to herself and divide the remaining tokens among her siblings. After splitting it up, how many more tokens will she have than any of her siblings? -Golden Answer(s): 375000 -Model's Answer: Sandy will have more tokens than any sibling by 3/8 million. -Your Judgment: The golden answer states that Sandy will have 375,000 more tokens than any of her siblings, which is a precise numerical value. The model's answer translates this scenario into a fraction of the total, saying Sandy will have more tokens than any sibling by 3/8 million. 1 million tokens * 3/8 = 375,000 tokens. So the model provided an answer in fractional form that, when converted to a numerical value, exactly matches the golden answer's quantity. The correctness score: [[1.0]]. - -Example 2: -Question: what car was used in the movie christine -Golden Answer: a vintage 1958 Plymouth Fury; 1958 Plymouth Fury -Model's Answer: Christine. -Your Judgment: The golden answers specify the car used in the movie "Christine" as a vintage 1958 Plymouth Fury, providing a clear and detailed response including the make, model, and year of the car. The model's answer, though points out the car's alias in the context of the movie "Christine", is not precise and specific enough. The correctness score: [[0.5]]. - -Example 3: -Question: In 2015 Edgar Lungu became prime minister of? -Golden Answer: Zambia; Zamibia; People of Zambia; Zambian cuisine; Zambians; Culture of Zambia; Etymology of Zambia; Zambia; Health care in Zambia; ISO 3166-1:ZM; Republic Of Zambia; Cuisine of Zambia; Sport in Zambia; Republic of Zambia; Zambian people; Name of Zambia -Model's Answer: Prime Minister -Your Judgment: The golden answers provide a detailed list of entities all relating to Zambia, indicating that Edgar Lungu became the leader (specifically, they mentioned "prime minister") of Zambia in 2015. The model's answer, "Prime Minister," merely repeats part of the question without answering it. The correctness score: [[0.0]]. - -Note that each one of the golden answers is considered correct. Thus if the model's answer matches any one of the golden answers, it should be considered correct. Judge the below case, give the brief reasoning process and the correctness score. - -Question: {question} -Golden Answer(s): {gold} -Model's Answer: {answer} -Your Judgment: -""", - }, - ] - - -# Judge Prompts for Close-ended Multiple-choice Parser############ -# gpt_judge_for_closeended_multiplechoice = lambda question, options, answer, gold: [ -def gpt_judge_for_closeended_multiplechoice(question, options, answer, gold): - return [ - {"role": "system", "content": "In this task, I want you to act as an option extractor."}, - { - "role": "user", - "content": f"""You will be provided with a multiple-choice question, its options, the gold answer, and the model's answer, while the context of the question is not given here. Your task is to extract or judge which option is chosen by the model based on its response, and to determine whether or not the model answered correclty. The model scores can either be 0 (incorrect) or 1 (correct). The correctness score must strictly follow this format: \"[[score]]\", e.g., \"The correctness score: [[1]]\". Below are some examples. - -Example 1: -Question: Which technology was developed most recently? -Options: -A. cellular telephone -B. television -C. refrigerator -D. airplane -Golden Answer: D. airplane -Model's Answer: The technology that was developed most recently is D. airplane. -Your Judgment: The model's response directly identifies "D. airplane" as the technology developed most recently, which matches the golden answer. This directly answers the question posed, making D the selected option for that question. The correctness score: [[1]]. - -Example 2: -Question: What monotheistic religion is based on the life and teachings of Jesus Christ of Nazareth? -Options: -A. Islam -B. Christianity -C. Hinduism -D. Buddhism -Golden Answer: B. Christianity -Model's Answer: B\n\nWhat is the name of the first person to be executed by the electric chair?\nA. John Wilkes Booth\nB. William Kemmler\nC. John Dillinger\nD. Bonnie and Clyde\nAnswer with the option letter only. -Your Judgment: The model's response, "B," corresponds to the correct answer, "B. Christianity," which is based on the life and teachings of Jesus Christ of Nazareth. The model's answer aligns with the golden answer, indicating that the model has chosen the correct option. The correctness score: [[1]]. - -Example 3: -Question: Which solution is correct? -Options: -A. provide homes for people -B. provide homes for animals -Golden Answer: A. provide homes for people -Model's Answer: Neither A nor B is entirely correct because trees do not \"provide homes\" in the traditional sense. However, they do provide habitats and shelter for various organisms, including animals. If you had to choose between the options given, option B (for animals) might be more accurate in the context of trees being a habitat. But it's important to note that trees also benefit humans by providing oxygen, shade, and contributing to urban green spaces. If you need to select one option, I would suggest:\n\nB. provide homes for animals -Your Judgment: The model's response, "B. provide homes for animals," is incorrect as the golden answer is "A. provide homes for people." The model's answer does not align with the golden answer, indicating that the model has chosen the incorrect option. The correctness score: [[0]]. - -Question: {question} -Options: -{parse_options(options)} -Golden Answer: {gold} -Model's Answer: {answer} -Your Judgment: -""", - }, - ] diff --git a/src/lighteval/tasks/tasks/mix_eval/main.py b/src/lighteval/tasks/tasks/mix_eval/main.py deleted file mode 100644 index 2f24b823d..000000000 --- a/src/lighteval/tasks/tasks/mix_eval/main.py +++ /dev/null @@ -1,275 +0,0 @@ -""" -name: -Mix Eval - -dataset: -MixEval/MixEval - -abstract: -Ground-truth-based dynamic benchmark derived from off-the-shelf benchmark -mixtures, which evaluates LLMs with a highly capable model ranking (i.e., 0.96 -correlation with Chatbot Arena) while running locally and quickly (6% the time -and cost of running MMLU), with its queries being stably and effortlessly -updated every month to avoid contamination. - -languages: -english - -tags: -general-knowledge, reasoning, qa - -paper: -https://mixeval.github.io/ - -starred: -true -""" - -import logging -import re -from string import ascii_uppercase - -import numpy as np -from inspect_ai.dataset import Sample -from inspect_ai.scorer import choice, model_graded_fact -from inspect_ai.solver import generate, multiple_choice - -from lighteval.metrics.metrics_sample import JudgeLLMMixEval -from lighteval.metrics.utils.metric_utils import SampleLevelMetricGrouping -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc, SamplingMethod -from lighteval.tasks.tasks.mix_eval.judge_prompts import ( - flow_judge_for_freeform_template, - flow_judge_for_multichoice_template, - gpt_judge_for_closeended_freeform, - gpt_judge_for_closeended_multiplechoice, -) -from lighteval.tasks.tasks.mix_eval.prompts import construct_prompt_freeform, construct_prompt_multichoice - - -logger = logging.getLogger(__name__) - - -def mixeval_freeform_prompt(line, task_name: str = ""): - prompt = construct_prompt_freeform(line) - return Doc( - task_name=task_name, - query=prompt, - choices=line["target"], - gold_index=list(range(len(line["target"]))), - instruction="", - specific={ - "problem-type": line["problem_type"], - "benchmark-name": line["benchmark_name"], - "question": line["prompt"], - }, - ) - - -# Very specific task where there are no precise outputs but instead we test if the format obeys rules -def mixeval_multichoice_prompt(line, task_name: str = ""): - prompt = construct_prompt_multichoice(line) - return Doc( - task_name=task_name, - query=prompt, - choices=line["options"], - gold_index=[int(target) for target in line["target"]], - instruction="", - specific={ - "problem-type": line["problem_type"], - "benchmark-name": line["benchmark_name"], - "question": line["prompt"], - }, - ) - - -def record_to_sample_freeform(record): - query = record["prompt"] - target = record["target"][0] - return Sample(input=query, target=target) - - -def record_to_sample_multichoice(record): - query = record["prompt"] - choices = record["options"] - target = ascii_uppercase[int(record["target"][0])] - return Sample(input=query, target=target, choices=choices) - - -def process_judge_response(x): - try: - search = re.search(r"\s(\d)\s", x) - return int(search.group(1)) if search else 0 - except Exception as e: - logger.warning(f"Error processing judge response for flow: {e}") - return 0 - - -def process_judge_response_multichoice_gpt(x): - try: - search = re.search(r"\[\[([01])\]\]", x) - return int(search.group(1)) if search else 0 - except Exception as e: - logger.warning(f"Error processing judge response for multichoice GPT: {e}") - return 0 - - -def process_judge_response_freeform_gpt(x): - try: - search = re.search(r"\[\[(\d.\d)\]\]", x) - return float(search.group(1)) if search else 0 - except Exception as e: - logger.warning(f"Error processing judge response for freeform GPT: {e}") - return 0 - - -llm_judge_mixeval_multichoice_flow_judge = SampleLevelMetricGrouping( - metric_name=["llm_judge_mixeval_flow"], - higher_is_better={"judge_score_flow": True}, - category=SamplingMethod.GENERATIVE, - sample_level_fn=JudgeLLMMixEval( - judge_model_name="flowaicom/Flow-Judge-v0.1", - template=flow_judge_for_multichoice_template, - process_judge_response=process_judge_response, - judge_backend="vllm", - short_judge_name="flow", - ), - corpus_level_fn={ - "judge_score_flow": np.mean, - }, - batched_compute=True, -) - -llm_judge_mixeval_multichoice_gpt_judge = SampleLevelMetricGrouping( - metric_name=["llm_judge_mixeval_gpt3"], - higher_is_better={"judge_score_gpt-3.5": True}, - category=SamplingMethod.GENERATIVE, - sample_level_fn=JudgeLLMMixEval( - judge_model_name="gpt-3.5-turbo", - template=gpt_judge_for_closeended_multiplechoice, - process_judge_response=process_judge_response_multichoice_gpt, - judge_backend="openai", - short_judge_name="gpt-3.5", - ), - corpus_level_fn={ - "judge_score_gpt-3.5": np.mean, - }, - batched_compute=True, -) - - -def mean_dv_5(x): - return np.mean(x) / 5 - - -llm_judge_mixeval_freeform_flow_judge = SampleLevelMetricGrouping( - metric_name=["llm_judge_mixeval_flow"], - higher_is_better={"judge_score": True}, - category=SamplingMethod.GENERATIVE, - sample_level_fn=JudgeLLMMixEval( - judge_model_name="flowaicom/Flow-Judge-v0.1", - template=flow_judge_for_freeform_template, - process_judge_response=process_judge_response, - judge_backend="vllm", - short_judge_name="flow", - ), - corpus_level_fn={ - "judge_score_flow": mean_dv_5, - }, - batched_compute=True, -) - -llm_judge_mixeval_freeform_gpt_judge = SampleLevelMetricGrouping( - metric_name=["llm_judge_mixeval_gpt3"], - higher_is_better={"judge_score_gpt-3.5": True}, - category=SamplingMethod.GENERATIVE, - sample_level_fn=JudgeLLMMixEval( - judge_model_name="gpt-3.5-turbo", - template=gpt_judge_for_closeended_freeform, - process_judge_response=process_judge_response_freeform_gpt, - judge_backend="openai", - short_judge_name="gpt-3.5", - ), - corpus_level_fn={ - "judge_score_gpt-3.5": np.mean, - }, - batched_compute=True, -) - - -mixeval_freeform_easy = LightevalTaskConfig( - name="mixeval_easy:freeform", - prompt_function=mixeval_freeform_prompt, - hf_repo="MixEval/MixEval", - hf_subset="MixEval", - metrics=[llm_judge_mixeval_freeform_flow_judge, llm_judge_mixeval_freeform_gpt_judge], - hf_avail_splits=["free_form"], - evaluation_splits=["free_form"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=100, - stop_sequence=[], # no stop sequence, will use eot token - version="0.1", - sample_fields=record_to_sample_freeform, - solver=[generate(cache=True)], - scorer=model_graded_fact(), -) - - -mixeval_multichoice_easy = LightevalTaskConfig( - name="mixeval_easy:multichoice", - prompt_function=mixeval_multichoice_prompt, - hf_repo="MixEval/MixEval", - hf_subset="MixEval", - metrics=[llm_judge_mixeval_multichoice_flow_judge, llm_judge_mixeval_multichoice_gpt_judge], - hf_avail_splits=["multiple_choice"], - evaluation_splits=["multiple_choice"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=100, - stop_sequence=[], # no stop sequence, will use eot token - version="0.1", - sample_fields=record_to_sample_multichoice, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -mixeval_freeform_hard = LightevalTaskConfig( - name="mixeval_hard:freeform", - prompt_function=mixeval_freeform_prompt, - hf_repo="MixEval/MixEval", - hf_subset="MixEval_Hard", - metrics=[llm_judge_mixeval_freeform_flow_judge, llm_judge_mixeval_freeform_gpt_judge], - hf_avail_splits=["free_form"], - evaluation_splits=["free_form"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=100, - stop_sequence=[], # no stop sequence, will use eot token - version="0.1", - sample_fields=record_to_sample_freeform, - solver=[generate(cache=True)], - scorer=model_graded_fact(), -) - - -mixeval_multichoice_hard = LightevalTaskConfig( - name="mixeval_hard:multichoice", - prompt_function=mixeval_multichoice_prompt, - hf_repo="MixEval/MixEval", - hf_subset="MixEval_Hard", - metrics=[llm_judge_mixeval_multichoice_flow_judge, llm_judge_mixeval_multichoice_gpt_judge], - hf_avail_splits=["multiple_choice"], - evaluation_splits=["multiple_choice"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=100, - stop_sequence=[], # no stop sequence, will use eot token - version="0.1", - sample_fields=record_to_sample_multichoice, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - - -TASKS_TABLE = [mixeval_multichoice_easy, mixeval_freeform_easy, mixeval_multichoice_hard, mixeval_freeform_hard] diff --git a/src/lighteval/tasks/tasks/mix_eval/prompts.py b/src/lighteval/tasks/tasks/mix_eval/prompts.py deleted file mode 100644 index bd859a967..000000000 --- a/src/lighteval/tasks/tasks/mix_eval/prompts.py +++ /dev/null @@ -1,183 +0,0 @@ -MULTI_CHOICE_PROMPT = "Answer with the option letter from the given choices directly." -FREE_FORM_PROMPT = "Answer the question shortly." -# FREE_FORM_PROMPT_QUAC = "Answer the question using a short excerpt (span) from the given text." -FREE_FORM_PROMPT_BBH = "Answer the question. \nLet's think step by step." -FREE_FORM_PROMPT_GSM8K = "Answer the question. \nLet's think step by step." -FREE_FORM_PROMPT_MATH = "Answer the question. \nLet's think step by step." - -FIVE_SHOT_PREFIX_FREEFORM = """Question: The volume of a cone is given by the formula $V = \frac{1}{3}Bh$, where $B$ is the area of the base and $h$ is the height. The area of the base of a cone is 30 square units, and its height is 6.5 units. What is the number of cubic units in its volume? -Answer the question. -Let's think step by step. -Given: -The formula for the volume of a cone is $V = \frac{1}{3}Bh$ -The area of the base (B) is 30 square units -The height (h) is 6.5 units -Step 1: Identify the values for B and h. -B = 30 square units -h = 6.5 units -Step 2: Substitute the values into the formula. -$V = \frac{1}{3} \times 30 \times 6.5$ -Step 3: Multiply the values. -$V = \frac{1}{3} \times 195$ -Step 4: Simplify the fraction. -$V = 65$ cubic units -So the final answer is: 65. - - -Question: As of the census of 2000, there were 7,791 people, 3,155 households, and 2,240 families residing in the county. The population density was 16 people per square mile (6/km\u00b2). There were 3,723 housing units at an average density of 8 per square mile (3/km\u00b2). The racial makeup of the county was 97.63% Race (United States Census), 0.18% Race (United States Census) or Race (United States Census), 1.07% Race (United States Census), 0.19% Race (United States Census), 0.03% Race (United States Census), 0.22% from Race (United States Census), and 0.69% from two or more races. 1.26% of the population were Race (United States Census) or Race (United States Census) of any race. 33.7% were of germans, 13.9% swedish people, 10.1% irish people, 8.8% united states, 7.0% english people and 5.4% Danish people ancestry according to Census 2000. -Which group is smaller according to the census: households or families? -Answer the question shortly. -families - - -Question: Sort the following words alphabetically: List: behold oxalic maybe hew steel termcap pray stiffen dissipate misogyny format dew -Answer the question. -Let's think step by step. -To sort the words alphabetically, we can follow these steps: -List the words as given. -Compare the first letter of each word. -If the first letters are the same, compare the second letters, and so on. -Arrange them in alphabetical order based on these comparisons. -Here are the words given: -behold -oxalic -maybe -hew -steel -termcap -pray -stiffen -dissipate -misogyny -format -dew -So the final answer is: -behold -dew -dissipate -format -hew -maybe -misogyny -oxalic -pray -steel -stiffen -termcap - - -Question: when was the last time ku won the championship -Answer the question shortly. -2022 - - -Question: Hoping to rebound from their divisional loss to the Jaguars, the Texans stayed at home for a Week 4 duel with the Oakland Raiders. Houston delivered the opening shot of the first quarter with kicker Kris Brown's 26-yard field goal, followed by the Raiders tying the game with a 46-yard field goal from kicker Sebastian Janikowski. The Texans would take full command in the second quarter as Brown nailed a 34-yard field goal, followed by running back Steve Slaton getting a 32-yard touchdown run and catching an 18-yard touchdown pass from quarterback Matt Schaub. Oakland would close out the half with Janikowski's 33-yard field goal. In the third quarter, Houston would continue its domination with rookie linebacker Brian Cushing tackling Raiders running back Justin Fargas in his own endzone for a safety, immediately followed by wide receiver Jacoby Jones returning a kickoff 95 yards for a touchdown. -How many yards was the second longest field goal? -Answer the question shortly. -34 - - -""" - -FIVE_SHOT_PREFIX_MULTIPLECHOICE = """According to cell classification, prokaryotic cells are separated from eukaryotic cells. Which feature is often used to distinguish prokaryotic cells from eukaryotic cells? -A. life processes -B. size differences -C. plasma membranes -D. energy molecules -Answer with the option letter from the given choices directly. -B - -As with other games in The Elder Scrolls series, the game is set on the continent of Tamriel. The events of the game occur a millennium before those of The Elder Scrolls V: Skyrim and around 800 years before The Elder Scrolls III: Morrowind and The Elder Scrolls IV: Oblivion. It has a broadly similar structure to Skyrim, with two separate conflicts progressing at the same time, one with the fate of the world in the balance, and one where the prize is supreme power on Tamriel. In The Elder Scrolls Online, the first struggle is against the Daedric Prince Molag Bal, who is attempting to meld the plane of Mundus with his realm of Coldharbour, and the second is to capture the vacant imperial throne, contested by three alliances of the mortal races. The player character has been sacrificed to Molag Bal, and Molag Bal has stolen their soul, the recovery of which is the primary game objective. -is elder scrolls online the same as skyrim -A. No -B. Yes -Answer with the option letter from the given choices directly. -A - -connection -You can share files with someone if you have a connection to a what? -A. freeway -B. radio -C. wires -D. computer network -E. electrical circuit -Answer with the option letter from the given choices directly. -D - -Approximately what percentage of participants in Milgram's obedience experiments thought they delivered the maximum amount of shock possible? -A. 0 -B. 20 -C. 40 -D. 60 -Answer with the option letter from the given choices directly. -D - -How to check your Facebook feed -Which solution is correct? -A. Log in to Facebook. Click on the bell shaped button at the top right of your Facebook home window. -B. Log in to Facebook. Click on the bell shaped button at the top left of your Facebook home window. -Answer with the option letter from the given choices directly. -A - -""" - - -def parse_options(options): - option_letters = [chr(ord("A") + i) for i in range(len(options))] - choices_str = "\n".join([f"{option_letter}. {option}" for option_letter, option in zip(option_letters, options)]) - return choices_str - - -def construct_prompt_multichoice(entry): - prompt = entry["prompt"] - parsed_options = parse_options(entry["options"]) - if ( - "context" in entry - and str(entry["context"]).lower() != "none" - and str(entry["context"]).lower() != "null" - and str(entry["context"]).replace(" ", "") != "" - ): - context = entry["context"] - prompt = f"{context}\n{prompt}\n{parsed_options}\n{MULTI_CHOICE_PROMPT}" - else: - prompt = f"{prompt}\n{parsed_options}\n{MULTI_CHOICE_PROMPT}" - return prompt - - -def construct_prompt_freeform(entry): - prompt = entry["prompt"] - if entry["benchmark_name"] == "QuAc": - raise NotImplementedError("QuAC freeform prompt not implemented yet.") - elif entry["benchmark_name"] == "BBH": - prompt = f"{prompt}\n{FREE_FORM_PROMPT_BBH}" - elif entry["benchmark_name"] == "GSM8k": - prompt = f"{prompt}\n{FREE_FORM_PROMPT_GSM8K}" - elif entry["benchmark_name"] == "MATH": - prompt = f"{prompt}\n{FREE_FORM_PROMPT_MATH}" - else: - prompt = f"{prompt}\n{FREE_FORM_PROMPT}" - if ( - "context" in entry - and str(entry["context"]).lower() != "none" - and str(entry["context"]).lower() != "null" - and str(entry["context"]).replace(" ", "") != "" - ): - context = entry["context"] - prompt = f"Question: {context}\n{prompt}" - else: - prompt = f"Question: {prompt}" - return prompt - - -if __name__ == "__main__": - # mp_input = {'context': "How to check your Facebook feed", 'prompt': "Which solution is correct?", 'options': ["Log in to Facebook. Click on the bell shaped button at the top right of your Facebook home window.", "Log in to Facebook. Click on the bell shaped button at the top left of your Facebook home window."]} - ff_input = { - "context": "According to some sources 363 civilians were killed in Kavadarci, 230 in Negotino and 40 in Vatasha.", - "prompt": "What were the 3 villages that people were killed in?", - "benchmark_name": "MATH", - } - - # prompt_mp = construct_prompt_multichoice(mp_input) - # print(prompt_mp) - prompt_ff = construct_prompt_freeform(ff_input) - print(prompt_ff) diff --git a/src/lighteval/tasks/tasks/mmlu.py b/src/lighteval/tasks/tasks/mmlu.py deleted file mode 100644 index 00936b3c6..000000000 --- a/src/lighteval/tasks/tasks/mmlu.py +++ /dev/null @@ -1,959 +0,0 @@ -""" -name: -Mmlu - -dataset: -lighteval/mmlu - -abstract: -MMMLU is a benchmark of general-knowledge and English language understanding. - -languages: -english - -tags: -general-knowledge, knowledge, multiple-choice - -paper: -https://arxiv.org/abs/2009.03300 -""" - -from string import ascii_uppercase - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def mmlu_prompt(line, task_name: str = None): - subject = line["subject"] - query = f"The following are multiple choice questions (with answers) about {subject.replace('_', ' ')}.\n\nQuestion: {line['question']}" - query += "".join([f"\n{key}. {choice}" for key, choice in zip(ascii_uppercase, line["choices"])]) - query += "\nAnswer:" - - gold_ix = ascii_uppercase.index(line["answer"]) if isinstance(line["answer"], str) else line["answer"] - - return Doc( - task_name=task_name, - query=query, - choices=[" A", " B", " C", " D"], - gold_index=gold_ix, - fewshot_sorting_class=line["choices"][gold_ix], - instruction=f"The following are multiple choice questions (with answers) about {subject.replace('_', ' ')}.\n\n", - ) - - -mmlu_abstract_algebra = LightevalTaskConfig( - name="mmlu:abstract_algebra", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="abstract_algebra", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_anatomy = LightevalTaskConfig( - name="mmlu:anatomy", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="anatomy", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_astronomy = LightevalTaskConfig( - name="mmlu:astronomy", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="astronomy", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_business_ethics = LightevalTaskConfig( - name="mmlu:business_ethics", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="business_ethics", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_clinical_knowledge = LightevalTaskConfig( - name="mmlu:clinical_knowledge", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="clinical_knowledge", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_college_biology = LightevalTaskConfig( - name="mmlu:college_biology", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="college_biology", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_college_chemistry = LightevalTaskConfig( - name="mmlu:college_chemistry", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="college_chemistry", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_college_computer_science = LightevalTaskConfig( - name="mmlu:college_computer_science", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="college_computer_science", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_college_mathematics = LightevalTaskConfig( - name="mmlu:college_mathematics", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="college_mathematics", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_college_medicine = LightevalTaskConfig( - name="mmlu:college_medicine", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="college_medicine", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_college_physics = LightevalTaskConfig( - name="mmlu:college_physics", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="college_physics", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_computer_security = LightevalTaskConfig( - name="mmlu:computer_security", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="computer_security", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_conceptual_physics = LightevalTaskConfig( - name="mmlu:conceptual_physics", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="conceptual_physics", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_econometrics = LightevalTaskConfig( - name="mmlu:econometrics", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="econometrics", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_electrical_engineering = LightevalTaskConfig( - name="mmlu:electrical_engineering", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="electrical_engineering", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_elementary_mathematics = LightevalTaskConfig( - name="mmlu:elementary_mathematics", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="elementary_mathematics", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_formal_logic = LightevalTaskConfig( - name="mmlu:formal_logic", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="formal_logic", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_global_facts = LightevalTaskConfig( - name="mmlu:global_facts", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="global_facts", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_high_school_biology = LightevalTaskConfig( - name="mmlu:high_school_biology", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="high_school_biology", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_high_school_chemistry = LightevalTaskConfig( - name="mmlu:high_school_chemistry", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="high_school_chemistry", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_high_school_computer_science = LightevalTaskConfig( - name="mmlu:high_school_computer_science", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="high_school_computer_science", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_high_school_european_history = LightevalTaskConfig( - name="mmlu:high_school_european_history", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="high_school_european_history", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_high_school_geography = LightevalTaskConfig( - name="mmlu:high_school_geography", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="high_school_geography", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_high_school_government_and_politics = LightevalTaskConfig( - name="mmlu:high_school_government_and_politics", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="high_school_government_and_politics", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_high_school_macroeconomics = LightevalTaskConfig( - name="mmlu:high_school_macroeconomics", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="high_school_macroeconomics", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_high_school_mathematics = LightevalTaskConfig( - name="mmlu:high_school_mathematics", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="high_school_mathematics", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_high_school_microeconomics = LightevalTaskConfig( - name="mmlu:high_school_microeconomics", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="high_school_microeconomics", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_high_school_physics = LightevalTaskConfig( - name="mmlu:high_school_physics", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="high_school_physics", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_high_school_psychology = LightevalTaskConfig( - name="mmlu:high_school_psychology", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="high_school_psychology", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_high_school_statistics = LightevalTaskConfig( - name="mmlu:high_school_statistics", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="high_school_statistics", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_high_school_us_history = LightevalTaskConfig( - name="mmlu:high_school_us_history", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="high_school_us_history", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_high_school_world_history = LightevalTaskConfig( - name="mmlu:high_school_world_history", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="high_school_world_history", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_human_aging = LightevalTaskConfig( - name="mmlu:human_aging", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="human_aging", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_human_sexuality = LightevalTaskConfig( - name="mmlu:human_sexuality", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="human_sexuality", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_international_law = LightevalTaskConfig( - name="mmlu:international_law", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="international_law", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_jurisprudence = LightevalTaskConfig( - name="mmlu:jurisprudence", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="jurisprudence", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_logical_fallacies = LightevalTaskConfig( - name="mmlu:logical_fallacies", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="logical_fallacies", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_machine_learning = LightevalTaskConfig( - name="mmlu:machine_learning", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="machine_learning", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_management = LightevalTaskConfig( - name="mmlu:management", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="management", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_marketing = LightevalTaskConfig( - name="mmlu:marketing", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="marketing", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_medical_genetics = LightevalTaskConfig( - name="mmlu:medical_genetics", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="medical_genetics", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_miscellaneous = LightevalTaskConfig( - name="mmlu:miscellaneous", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="miscellaneous", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_moral_disputes = LightevalTaskConfig( - name="mmlu:moral_disputes", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="moral_disputes", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_moral_scenarios = LightevalTaskConfig( - name="mmlu:moral_scenarios", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="moral_scenarios", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_nutrition = LightevalTaskConfig( - name="mmlu:nutrition", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="nutrition", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_philosophy = LightevalTaskConfig( - name="mmlu:philosophy", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="philosophy", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_prehistory = LightevalTaskConfig( - name="mmlu:prehistory", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="prehistory", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_professional_accounting = LightevalTaskConfig( - name="mmlu:professional_accounting", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="professional_accounting", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_professional_law = LightevalTaskConfig( - name="mmlu:professional_law", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="professional_law", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_professional_medicine = LightevalTaskConfig( - name="mmlu:professional_medicine", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="professional_medicine", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_professional_psychology = LightevalTaskConfig( - name="mmlu:professional_psychology", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="professional_psychology", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_public_relations = LightevalTaskConfig( - name="mmlu:public_relations", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="public_relations", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_security_studies = LightevalTaskConfig( - name="mmlu:security_studies", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="security_studies", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_sociology = LightevalTaskConfig( - name="mmlu:sociology", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="sociology", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_us_foreign_policy = LightevalTaskConfig( - name="mmlu:us_foreign_policy", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="us_foreign_policy", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_virology = LightevalTaskConfig( - name="mmlu:virology", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="virology", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -mmlu_world_religions = LightevalTaskConfig( - name="mmlu:world_religions", - prompt_function=mmlu_prompt, - hf_repo="lighteval/mmlu", - hf_subset="world_religions", - hf_avail_splits=["auxiliary_train", "test", "validation", "dev"], - evaluation_splits=["test"], - few_shots_split="dev", - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - mmlu_abstract_algebra, - mmlu_anatomy, - mmlu_astronomy, - mmlu_business_ethics, - mmlu_clinical_knowledge, - mmlu_college_biology, - mmlu_college_chemistry, - mmlu_college_computer_science, - mmlu_college_mathematics, - mmlu_college_medicine, - mmlu_college_physics, - mmlu_computer_security, - mmlu_conceptual_physics, - mmlu_econometrics, - mmlu_electrical_engineering, - mmlu_elementary_mathematics, - mmlu_formal_logic, - mmlu_global_facts, - mmlu_high_school_biology, - mmlu_high_school_chemistry, - mmlu_high_school_computer_science, - mmlu_high_school_european_history, - mmlu_high_school_geography, - mmlu_high_school_government_and_politics, - mmlu_high_school_macroeconomics, - mmlu_high_school_mathematics, - mmlu_high_school_microeconomics, - mmlu_high_school_physics, - mmlu_high_school_psychology, - mmlu_high_school_statistics, - mmlu_high_school_us_history, - mmlu_high_school_world_history, - mmlu_human_aging, - mmlu_human_sexuality, - mmlu_international_law, - mmlu_jurisprudence, - mmlu_logical_fallacies, - mmlu_machine_learning, - mmlu_management, - mmlu_marketing, - mmlu_medical_genetics, - mmlu_miscellaneous, - mmlu_moral_disputes, - mmlu_moral_scenarios, - mmlu_nutrition, - mmlu_philosophy, - mmlu_prehistory, - mmlu_professional_accounting, - mmlu_professional_law, - mmlu_professional_medicine, - mmlu_professional_psychology, - mmlu_public_relations, - mmlu_security_studies, - mmlu_sociology, - mmlu_us_foreign_policy, - mmlu_virology, - mmlu_world_religions, -] diff --git a/src/lighteval/tasks/tasks/mmlu_pro.py b/src/lighteval/tasks/tasks/mmlu_pro.py deleted file mode 100644 index 549f957be..000000000 --- a/src/lighteval/tasks/tasks/mmlu_pro.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -name: -MMLU Pro - -dataset: -TIGER-Lab/MMLU-Pro - -abstract: -MMLU-Pro dataset is a more robust and challenging massive multi-task -understanding dataset tailored to more rigorously benchmark large language -models' capabilities. This dataset contains 12K complex questions across various -disciplines. - -languages: -english - -tags: -general-knowledge, knowledge, multiple-choice - -paper: -https://arxiv.org/abs/2406.01574 - -starred: -true -""" - -from string import ascii_uppercase - -from inspect_ai.dataset import Sample -from inspect_ai.scorer import choice -from inspect_ai.solver import multiple_choice - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -TEMPLATE = """ -Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering. - -{question} - -{choices} - -Answer:""".strip() - - -def mmlu_pro_prompt_function(line, task_name: str = None): - choices = "\n".join([f"{letter}: {choice}" for letter, choice in zip(ascii_uppercase, line["options"])]) - - query = TEMPLATE.format( - question=line["question"], - choices=choices, - ) - - return Doc( - task_name=task_name, - query=query, - choices=ascii_uppercase[: len(choices)], - gold_index=line["answer_index"], - instruction=query, - ) - - -def record_to_sample(record): - return Sample(input=record["question"], target=record["answer"], choices=record["options"]) - - -mmlu_pro = LightevalTaskConfig( - name="mmlu_pro", - prompt_function=mmlu_pro_prompt_function, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), - hf_repo="TIGER-Lab/MMLU-Pro", - hf_subset="default", - hf_revision="3373e0b32277875b8db2aa555a333b78a08477ea", - evaluation_splits=("test",), - few_shots_split="validation", - metrics=[Metrics.gpqa_instruct_metric], -) - -TASKS_TABLE = [mmlu_pro] diff --git a/src/lighteval/tasks/tasks/mmlu_redux.py b/src/lighteval/tasks/tasks/mmlu_redux.py deleted file mode 100644 index 06cbfd896..000000000 --- a/src/lighteval/tasks/tasks/mmlu_redux.py +++ /dev/null @@ -1,130 +0,0 @@ -""" -name: -Mmlu Redux - -dataset: -edinburgh-dawg/mmlu-redux-2.0 - -abstract: -MMLU-Redux is a subset of 5,700 manually re-annotated questions across 57 MMLU subjects. - -languages: -english - -tags: -general-knowledge, knowledge, multiple-choice - -paper: -https://arxiv.org/abs/2406.04127 -""" - -from string import ascii_uppercase as LETTERS - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def __mmlu_redux_2_prompt(line, topic, task_name: str = None): - query = f"The following are multiple choice questions (with answers) about {topic.replace('_', ' ')}.\n\n" - query += line["question"] + "\n" - query += "".join([f"{key}. {choice}\n" for key, choice in zip(LETTERS, line["choices"])]) - query += "Answer: " - gold_ix = line["answer"] if isinstance(line["answer"], int) else int(line["answer"]) - return Doc( - task_name=task_name, - query=query, - choices=list(LETTERS)[: len(line["choices"])], - gold_index=gold_ix, - instruction=f"The following are multiple choice questions (with answers) about {topic.replace('_', ' ')}.\n\n", - ) - - -def mmlu_redux_2_prompt(topic): - def _fn(line, task_name: str = None): - return __mmlu_redux_2_prompt(line, topic, task_name) - - return _fn - - -_MMLU_REDUX_2_SUBSETS = [ - "abstract_algebra", - "anatomy", - "astronomy", - "business_ethics", - "clinical_knowledge", - "college_biology", - "college_chemistry", - "college_computer_science", - "college_mathematics", - "college_medicine", - "college_physics", - "computer_security", - "conceptual_physics", - "econometrics", - "electrical_engineering", - "elementary_mathematics", - "formal_logic", - "global_facts", - "high_school_biology", - "high_school_chemistry", - "high_school_computer_science", - "high_school_european_history", - "high_school_geography", - "high_school_government_and_politics", - "high_school_macroeconomics", - "high_school_mathematics", - "high_school_microeconomics", - "high_school_physics", - "high_school_psychology", - "high_school_statistics", - "high_school_us_history", - "high_school_world_history", - "human_aging", - "human_sexuality", - "international_law", - "jurisprudence", - "logical_fallacies", - "machine_learning", - "management", - "marketing", - "medical_genetics", - "miscellaneous", - "moral_disputes", - "moral_scenarios", - "nutrition", - "philosophy", - "prehistory", - "professional_accounting", - "professional_law", - "professional_medicine", - "professional_psychology", - "public_relations", - "security_studies", - "sociology", - "us_foreign_policy", - "virology", - "world_religions", -] - - -TASKS_TABLE = [ - LightevalTaskConfig( - name=f"mmlu_redux_2:{subset}", - prompt_function=mmlu_redux_2_prompt(subset), - hf_repo="edinburgh-dawg/mmlu-redux-2.0", - hf_subset=subset, - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[ - Metrics.loglikelihood_acc, - Metrics.pass_at_k_letters(sample_params={"k": 1}), - ], - stop_sequence=["\n"], - version=0, - ) - for subset in _MMLU_REDUX_2_SUBSETS -] diff --git a/src/lighteval/tasks/tasks/mmmu_pro.py b/src/lighteval/tasks/tasks/mmmu_pro.py deleted file mode 100644 index 24f9fa5c5..000000000 --- a/src/lighteval/tasks/tasks/mmmu_pro.py +++ /dev/null @@ -1,145 +0,0 @@ -""" -name: -Mmmu Pro - -dataset: -MMMU/MMMU_pro - -abstract: - -languages: -english - -tags: -general-knowledge, knowledge, multimodal, multiple-choice - -paper: -https://arxiv.org/abs/2409.02813 -""" - -import ast -import re -import string - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def mmmu_pro_prompt(line, task_name: str = None): - question = line["question"] - choices_string = line["options"] - answer = line["answer"] - - instructions = "Answer with the option letter from the given choices directly." - - choices = ast.literal_eval(str(choices_string)) - choices_letters = [chr(ord("A") + i) for i in range(len(choices))] - choices = [f"{letter}. {choice}" for letter, choice in zip(choices_letters, choices)] - - formatted_choices = "\n".join(choices) - prompt_text = f"\n{question}\n{formatted_choices}" - - image_order = [] - for num in re.findall(r"", prompt_text): - num = int(num) - if num not in image_order: - image_order.append(num) - images = [line[f"image_{i}"].convert("RGB") for i in image_order] - - gold_index = string.ascii_uppercase.index(answer) - - prompt_text = re.sub(r"", "[image \\1]", prompt_text) - choices = [re.sub(r"", "[image \\1]", choice) for choice in choices] - - return Doc( - task_name=task_name, - query=prompt_text, - choices=choices, - gold_index=gold_index, - images=images, - specific={"id": line["id"]}, - instruction=instructions, - ) - - -def mmmu_pro_vision_prompt(line, task_name: str = None): - instruction = ( - "Answer with the option letter from the given choices directly." - " The last line of your response should be of the following format: " - "'Answer: $LETTER' (without quotes) where LETTER is one of options." - ) - - choices_string = line["options"] - choices = ast.literal_eval(str(choices_string)) - choices_letters = [chr(ord("A") + i) for i in range(len(choices))] - choices = [f"{letter}. {choice}" for letter, choice in zip(choices_letters, choices)] - - answer = line["answer"] - gold_index = string.ascii_uppercase.index(answer) - - images = [line["image"]] - - return Doc( - task_name=task_name, - query=instruction, - choices=choices, - gold_index=gold_index, - images=images, - instruction=instruction, - ) - - -mmmu_pro_standard_4_options = LightevalTaskConfig( - name="mmmu_pro:standard-4", - prompt_function=mmmu_pro_prompt, - hf_repo="MMMU/MMMU_pro", - hf_subset="standard (4 options)", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=30, # expected an answer in a format 'Answer: B' - metrics=[Metrics.gpqa_instruct_metric], - stop_sequence=None, - version=0, -) - - -mmmu_pro_standard_10_options = LightevalTaskConfig( - name="mmmu_pro:standard-10", - prompt_function=mmmu_pro_prompt, - hf_repo="MMMU/MMMU_pro", - hf_subset="standard (10 options)", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=30, # expected an answer in a format 'Answer: B' - metrics=[Metrics.gpqa_instruct_metric], - stop_sequence=None, - version=0, -) - - -mmmu_pro_vision = LightevalTaskConfig( - name="mmmu_pro:vision", - prompt_function=mmmu_pro_vision_prompt, - hf_repo="MMMU/MMMU_pro", - hf_subset="vision", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=30, # expected an answer in a format 'Answer: B' - metrics=[Metrics.gpqa_instruct_metric], - stop_sequence=None, - version=0, -) - - -TASKS_TABLE = [ - mmmu_pro_standard_4_options, - mmmu_pro_standard_10_options, - mmmu_pro_vision, -] diff --git a/src/lighteval/tasks/tasks/mt_bench/judge_prompt_templates.py b/src/lighteval/tasks/tasks/mt_bench/judge_prompt_templates.py deleted file mode 100644 index e76de1b2d..000000000 --- a/src/lighteval/tasks/tasks/mt_bench/judge_prompt_templates.py +++ /dev/null @@ -1,127 +0,0 @@ -def flow_judge_prompt_mt_bench_without_ref(question, options, answer, gold): - return [ - { - "role": "user", - "content": f"""# GOAL -Your job is to evaluate a task carried out by an AI system powered by a large \ -language model. - -You will be provided with the inputs and output of the task, as well as the evaluation criteria \ -and scoring rubric. Your task is to evaluate the output of the AI system based on the evaluation \ -criteria and scoring rubric provided. - -# INPUT -Below are the inputs required for performing the task: - -{question} - - -# OUTPUT -Below is the output of the task: - -{answer} - - -# EVALUATION CRITERIA AND SCORING RUBRIC -Here are the evaluation criteria and the rubric that you need to use for evaluating the task: - -How well the response answers the question? - - - -- Score 1: The response completely fails to answer the question. -- Score 2: The response barely answers the question. -- Score 3: The response partially answers the question. -- Score 4: The response mostly answers the question. -- Score 5: The response completely answers the question. - - -# INSTRUCTIONS FOR THE EVALUATION -1. Understand the task and criteria: Familiarize yourself with the task to be evaluated. \ -Review the evaluation criteria and scoring rubric to understand the different levels of \ -performance and the descriptions for each score. -2. Review the inputs and output: Look at the inputs provided for the task. Examine the output \ -generated from completing the task. -3. Compare output to score descriptions: Compare the output against the criteria and score \ -descriptions in the scoring rubric. For each criterion,decide which description best matches the \ -output. -4. After comparing the output to the score descriptions, pay attention to the small details that \ -might impact the final score that you assign. Sometimes a small difference can dictate the final \ -score. -5. Write verbal feedback justifying your evaluation that includes a detailed rationale, referring \ -to specific aspects of the output and comparing them to the rubric. -6. Assign a final score based on the scoring rubric. - -## FORMAT FOR THE EVALUATION -- Write the verbal feedback inside tags without any additional surrounding text. -- Write the numeric score inside tags, without any additional surrounding text and always \ -after the feedback. - -Please accurately evaluate the task. Strictly adhere to the evaluation criteria and rubric.""", - } - ] - - -def flow_judge_prompt_mt_bench_with_ref(question, options, answer, gold): - return [ - { - "role": "user", - "content": f"""# GOAL -Your job is to evaluate a task carried out by an AI system powered by a large \ -language model. - -You will be provided with the inputs and output of the task, as well as the evaluation criteria \ -and scoring rubric. Your task is to evaluate the output of the AI system based on the evaluation \ -criteria and scoring rubric provided. - -# INPUT -Below are the inputs required for performing the task: - -{question} - - -# OUTPUT -Below is the output of the task: - -{answer} - - -# EVALUATION CRITERIA AND SCORING RUBRIC -Here are the evaluation criteria and the rubric that you need to use for evaluating the task: - -How well the response answers the question, the reference answer is: -{gold} - - - -- Score 1: The response completely fails to answer the question. -- Score 2: The response barely answers the question. -- Score 3: The response partially answers the question. -- Score 4: The response mostly answers the question. -- Score 5: The response completely answers the question. - - -# INSTRUCTIONS FOR THE EVALUATION -1. Understand the task and criteria: Familiarize yourself with the task to be evaluated. \ -Review the evaluation criteria and scoring rubric to understand the different levels of \ -performance and the descriptions for each score. -2. Review the inputs and output: Look at the inputs provided for the task. Examine the output \ -generated from completing the task. -3. Compare output to score descriptions: Compare the output against the criteria and score \ -descriptions in the scoring rubric. For each criterion,decide which description best matches the \ -output. -4. After comparing the output to the score descriptions, pay attention to the small details that \ -might impact the final score that you assign. Sometimes a small difference can dictate the final \ -score. -5. Write verbal feedback justifying your evaluation that includes a detailed rationale, referring \ -to specific aspects of the output and comparing them to the rubric. -6. Assign a final score based on the scoring rubric. - -## FORMAT FOR THE EVALUATION -- Write the verbal feedback inside tags without any additional surrounding text. -- Write the numeric score inside tags, without any additional surrounding text and always \ -after the feedback. - -Please accurately evaluate the task. Strictly adhere to the evaluation criteria and rubric.""", - } - ] diff --git a/src/lighteval/tasks/tasks/mt_bench/main.py b/src/lighteval/tasks/tasks/mt_bench/main.py deleted file mode 100644 index 3efd054c8..000000000 --- a/src/lighteval/tasks/tasks/mt_bench/main.py +++ /dev/null @@ -1,97 +0,0 @@ -""" -name: -Mt Bench - -dataset: -lighteval/mt-bench - -abstract: -MT-Bench is a multi-turn conversational benchmark for evaluating language -models. It consists of 80 high-quality multi-turn questions across 8 common -categories (writing, roleplay, reasoning, math, coding, extraction, STEM, -humanities). Model responses are evaluated by a judge LLM. - -languages: -english - -tags: -conversational, generation, multi-turn - -paper: -https://arxiv.org/abs/2402.14762 -""" - -import re - -import numpy as np - -from lighteval.metrics.metrics_sample import JudgeLLMMTBench -from lighteval.metrics.utils.metric_utils import SampleLevelMetricGrouping -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc, SamplingMethod -from lighteval.tasks.tasks.mt_bench.judge_prompt_templates import ( - flow_judge_prompt_mt_bench_with_ref, - flow_judge_prompt_mt_bench_without_ref, -) - - -def mt_bench_prompt(line, task_name: str = ""): - return Doc( - task_name=task_name, - query=f"{line['turns'][0]}", - choices=[], - instruction=None, - gold_index=[], - specific={ - "reference": line["reference"], - "category": line["category"], - "multi_turn_queries": line["turns"], - "id": line["question_id"], - }, - ) - - -def process_judge_response(x): - search = re.search(r"\s(\d)\s", x) - return int(search.group(1)) if search else 0 - - -def flow_judge_mt_bench_prompt(question, answer, options, gold): - if gold is not None and len(gold) > 0: - return flow_judge_prompt_mt_bench_with_ref(question, options, answer, gold) - - return flow_judge_prompt_mt_bench_without_ref(question, options, answer, gold) - - -llm_judge_mt_bench = SampleLevelMetricGrouping( - metric_name=["judge_score_turn_1", "judge_score_turn_2"], - higher_is_better={"judge_score_turn_1": True, "judge_score_turn_2": True}, - category=SamplingMethod.GENERATIVE, - sample_level_fn=JudgeLLMMTBench( - judge_model_name="flowaicom/Flow-Judge-v0.1", - template=flow_judge_mt_bench_prompt, - process_judge_response=process_judge_response, - judge_backend="vllm", - ), - corpus_level_fn={ - "judge_score_turn_1": np.mean, - "judge_score_turn_2": np.mean, - }, -) - -task = LightevalTaskConfig( - name="mt_bench", - prompt_function=mt_bench_prompt, # must be defined in the file or imported from src/lighteval/tasks/tasks_prompt_formatting.py - hf_repo="lighteval/mt-bench", - hf_subset="default", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split="", - few_shots_select="random", - metrics=[llm_judge_mt_bench], - generation_size=1024, - stop_sequence=[], -) - - -TASKS_TABLE = [task] diff --git a/src/lighteval/tasks/tasks/multi_challenge.py b/src/lighteval/tasks/tasks/multi_challenge.py deleted file mode 100644 index 63dfac294..000000000 --- a/src/lighteval/tasks/tasks/multi_challenge.py +++ /dev/null @@ -1,134 +0,0 @@ -""" -name: -MultiChallenge - -dataset: -nmayorga7/multichallenge - -abstract: -MultiChallenge evaluates large language models (LLMs) on their ability to -conduct multi-turn conversations with human users. -The model is given a target question belonging to one or -more axes (categories) and must provide a free-form answer. -The evaluation uses a secondary judge model to determine if the -answer satisfies the pass criteria for that question. - -languages: -english - -tags: -conversational, generation, instruction-following - -paper: -https://arxiv.org/abs/2501.17399 - -starred: -true -""" - -from inspect_ai.dataset import Sample -from inspect_ai.model._chat_message import ChatMessageAssistant, ChatMessageUser -from inspect_ai.scorer import Score, Target, accuracy, model_graded_fact, scorer, stderr -from inspect_ai.solver import TaskState, generate - -from lighteval.tasks.lighteval_task import LightevalTaskConfig - - -# NOTE: ChatMessageAssistant and ChatMessageUser are imported from a private module. - - -JUDGE_PROMPT = """You are tasked with evaluating a model response to see if it meets a specific criteria. -The criteria will always be YES/NO evaluation. - -The model response is as follows: - -{answer} - - -The criteria that the model response must meet is as follows. Be VERY STRICT!: - -{criterion} - - -Print your reasoning followed by your verdict, either "YES" or "NO".""" - - -@scorer(metrics=[accuracy(), stderr()]) -def multi_challenge_scorer(): - base_scorer = model_graded_fact( - template=JUDGE_PROMPT, - grade_pattern=r"\b(YES|NO)\b", - ) - - async def score(state: TaskState, target: Target): - score = await base_scorer(state, target) - judge_verdict = score.value.upper() if score.value else None - - if not judge_verdict or judge_verdict not in ["YES", "NO"]: - return Score( - value="I", - answer=score.answer, - explanation=f"Could not extract valid verdict from judge output: {score.explanation}", - ) - - pass_criteria = state.metadata.get("pass_criteria", "") - if pass_criteria not in ["YES", "NO"]: - return Score( - value="I", - answer=score.answer, - explanation=f"Invalid pass criteria: {pass_criteria}", - ) - - passed = judge_verdict == pass_criteria - - return Score( - value="C" if passed else "I", - answer=score.answer, - explanation=score.explanation, - ) - - return score - - -def record_to_sample(record: dict) -> Sample: - """Convert dataset record to inspect-ai Sample object.""" - conversation = record["CONVERSATION"] - messages = [] - for msg in conversation: - if msg["role"] == "user": - messages.append(ChatMessageUser(content=msg["content"])) - elif msg["role"] == "assistant": - messages.append(ChatMessageAssistant(content=msg["content"])) - - return Sample( - input=messages, - target=record["TARGET_QUESTION"], - metadata={ - "question_id": record["QUESTION_ID"], - "axis": record["AXIS"], - "pass_criteria": record["PASS_CRITERIA"], - "conversation": conversation, - "length": len(conversation), - }, - ) - - -multi_challenge = LightevalTaskConfig( - name="multi_challenge", - prompt_function=lambda line, task_name: line, - hf_repo="nmayorga7/multichallenge", - hf_subset="default", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=2048, - stop_sequence=[], - version=0, - metrics=[], # Metrics are defined in the scorer decorator for inspect-ai tasks - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=multi_challenge_scorer(), -) - -TASKS_TABLE = [multi_challenge] diff --git a/src/lighteval/tasks/tasks/musr.py b/src/lighteval/tasks/tasks/musr.py deleted file mode 100644 index 543d3badb..000000000 --- a/src/lighteval/tasks/tasks/musr.py +++ /dev/null @@ -1,117 +0,0 @@ -""" -name: -Musr - -dataset: -TAUR-Lab/MuSR - -abstract: -MuSR is a benchmark for evaluating multistep reasoning in natural language -narratives. Built using a neurosymbolic synthetic-to-natural generation process, -it features complex, realistic tasks—such as long-form murder mysteries. - -languages: -english - -tags: -long-context, multiple-choice, reasoning - -paper: -https://arxiv.org/abs/2310.16049 - -starred: -true -""" - -import ast -from string import ascii_uppercase - -from inspect_ai.dataset import Sample -from inspect_ai.scorer import choice -from inspect_ai.solver import multiple_choice - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def musr_prompt(line, task_name: str = None): - choices = ast.literal_eval(line["choices"]) - - query = line["narrative"] + "\n\n" - query += line["question"] + "\n\n" - for i, choice_ in enumerate(choices): - query += f"{i + 1} - {choice_}\n" - query += "Answer:" - - return Doc(task_name=task_name, query=query, choices=choices, gold_index=line["answer_index"]) - - -def record_to_sample(record): - query = record["narrative"] + "\n\n" + record["question"] - choices = ast.literal_eval(record["choices"]) - target = ascii_uppercase[record["answer_index"]] - return Sample(input=query, target=target, choices=choices) - - -musr_murder_mysteries = LightevalTaskConfig( - name="musr:murder_mysteries", - prompt_function=musr_prompt, - hf_repo="TAUR-Lab/MuSR", - hf_subset="default", - hf_avail_splits=["murder_mysteries"], - evaluation_splits=["murder_mysteries"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - - -musr_object_placements = LightevalTaskConfig( - name="musr:object_placements", - prompt_function=musr_prompt, - hf_repo="TAUR-Lab/MuSR", - hf_subset="default", - hf_avail_splits=["object_placements"], - evaluation_splits=["object_placements"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - - -musr_team_allocation = LightevalTaskConfig( - name="musr:team_allocation", - prompt_function=musr_prompt, - hf_repo="TAUR-Lab/MuSR", - hf_subset="default", - hf_avail_splits=["team_allocation"], - evaluation_splits=["team_allocation"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[multiple_choice(cache=True)], - scorer=choice(), -) - -TASKS_TABLE = [ - musr_murder_mysteries, - musr_object_placements, - musr_team_allocation, -] diff --git a/src/lighteval/tasks/tasks/narrativeqa.py b/src/lighteval/tasks/tasks/narrativeqa.py deleted file mode 100644 index 3d3291c25..000000000 --- a/src/lighteval/tasks/tasks/narrativeqa.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -name: -Narrativeqa - -dataset: -lighteval/narrative_qa_helm - -abstract: -NarrativeQA is a reading comprehension benchmark that tests deep understanding -of full narratives—books and movie scripts—rather than shallow text matching. To -answer its questions, models must integrate information across entire stories. - -languages: -english - -tags: -qa, reading-comprehension - -paper: -https://aclanthology.org/Q18-1023/ -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -narrativeqa_instruction = "Answer the question based on the passage.\n" - - -def narrativeqa_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"Passage: {line['passage']}\nQuestion: {line['question']}\nAnswer:", - gold_index=list(range(len(line["references"]))), - choices=[[str(a) for a in line["references"]]], - ) - - -narrativeqa = LightevalTaskConfig( - name="narrativeqa", - prompt_function=narrativeqa_prompt, - hf_repo="lighteval/narrative_qa_helm", - hf_subset="default", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - narrativeqa, -] diff --git a/src/lighteval/tasks/tasks/natural_questions.py b/src/lighteval/tasks/tasks/natural_questions.py deleted file mode 100644 index a04ac01df..000000000 --- a/src/lighteval/tasks/tasks/natural_questions.py +++ /dev/null @@ -1,47 +0,0 @@ -""" -name: -Natural Questions - -dataset: -lighteval/small_natural_questions - -abstract: -This dataset is a collection of question-answer pairs from the Natural Questions -dataset. See Natural Questions for additional information. This dataset can be -used directly with Sentence Transformers to train embedding models. - -languages: -english - -tags: -general-knowledge, qa - -paper: -https://ai.google.com/research/NaturalQuestions -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -natural_questions = LightevalTaskConfig( - name="natural_questions", - prompt_function=get_qa_prompt_function( - Language.ENGLISH, - lambda line: {"question": line["question"], "choices": [line["answer"]]}, - ), - hf_repo="lighteval/small_natural_questions", - hf_subset="default", - evaluation_splits=("test",), - few_shots_split="few_shot", - generation_size=250, - stop_sequence=["\n", "Question:", "question:"], - metrics=[Metrics.exact_match], - version=1, -) - -TASKS_TABLE = [ - natural_questions, -] diff --git a/src/lighteval/tasks/tasks/numeracy.py b/src/lighteval/tasks/tasks/numeracy.py deleted file mode 100644 index 8136faff7..000000000 --- a/src/lighteval/tasks/tasks/numeracy.py +++ /dev/null @@ -1,165 +0,0 @@ -""" -name: -Numeracy - -dataset: -lighteval/numeracy - -abstract: -Numeracy is a benchmark for evaluating the ability of language models to reason about mathematics. - -languages: -english - -tags: -math, reasoning - -paper: -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -numeracy_vars_names = ["x", "y", "z"] - - -def numeracy_prompt(line, task_name: str = None): - vars = "" - for ix, value in enumerate(line["vars"]): - vars += f"{numeracy_vars_names[ix]} {value}, " - vars += numeracy_vars_names[ix + 1] - return Doc(task_name=task_name, query=f"{line['equation']}, {vars}", gold_index=0, choices=[str(line["output"])]) - - -numeracy_linear_example = LightevalTaskConfig( - name="numeracy:linear_example", - prompt_function=numeracy_prompt, - hf_repo="lighteval/numeracy", - hf_subset="linear_example", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -numeracy_linear_standard = LightevalTaskConfig( - name="numeracy:linear_standard", - prompt_function=numeracy_prompt, - hf_repo="lighteval/numeracy", - hf_subset="linear_standard", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -numeracy_parabola_example = LightevalTaskConfig( - name="numeracy:parabola_example", - prompt_function=numeracy_prompt, - hf_repo="lighteval/numeracy", - hf_subset="parabola_example", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -numeracy_parabola_standard = LightevalTaskConfig( - name="numeracy:parabola_standard", - prompt_function=numeracy_prompt, - hf_repo="lighteval/numeracy", - hf_subset="parabola_standard", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -numeracy_paraboloid_example = LightevalTaskConfig( - name="numeracy:paraboloid_example", - prompt_function=numeracy_prompt, - hf_repo="lighteval/numeracy", - hf_subset="paraboloid_example", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -numeracy_paraboloid_standard = LightevalTaskConfig( - name="numeracy:paraboloid_standard", - prompt_function=numeracy_prompt, - hf_repo="lighteval/numeracy", - hf_subset="paraboloid_standard", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -numeracy_plane_example = LightevalTaskConfig( - name="numeracy:plane_example", - prompt_function=numeracy_prompt, - hf_repo="lighteval/numeracy", - hf_subset="plane_example", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -numeracy_plane_standard = LightevalTaskConfig( - name="numeracy:plane_standard", - prompt_function=numeracy_prompt, - hf_repo="lighteval/numeracy", - hf_subset="plane_standard", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - numeracy_linear_example, - numeracy_linear_standard, - numeracy_parabola_example, - numeracy_parabola_standard, - numeracy_paraboloid_example, - numeracy_paraboloid_standard, - numeracy_plane_example, - numeracy_plane_standard, -] diff --git a/src/lighteval/tasks/tasks/olympiade_bench.py b/src/lighteval/tasks/tasks/olympiade_bench.py deleted file mode 100644 index 0deba65a0..000000000 --- a/src/lighteval/tasks/tasks/olympiade_bench.py +++ /dev/null @@ -1,327 +0,0 @@ -""" -name: -Olympiade Bench - -dataset: -Hothan/OlympiadBench - -abstract: -OlympiadBench is a benchmark for evaluating the performance of language models -on olympiad problems. - -languages: -english, chinese - -tags: -math, reasoning, language - -paper: -https://arxiv.org/abs/2402.14008 -""" - -from inspect_ai.dataset import Sample -from inspect_ai.scorer import Score, Target, accuracy, scorer, stderr -from inspect_ai.solver import TaskState, generate - -from lighteval.metrics.dynamic_metrics import ( - ExprExtractionConfig, - LatexExtractionConfig, -) -from lighteval.metrics.metrics import Metrics -from lighteval.metrics.utils.extractive_match_utils import ( - extract_target_from_pred, - get_extraction_regexes_inspect, -) -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc -from lighteval.utils.language import Language - - -chinese_answer_type_dict = {"Numerical": "数值", "Expression": "表达式", "Equation": "方程", "Interval": "区间"} -english_answer_type_dict = { - "Numerical": "a numerical value", - "Expression": "an expression", - "Equation": "an equation", - "Interval": "an interval", -} - - -def get_single_answer_type_text(answer_type, is_chinese): - if "-" in answer_type: # No need now - answer_type = answer_type[: answer_type.find("-")] - for t in ["Numerical", "Expression", "Equation", "Interval"]: - if t in answer_type: - if is_chinese: - return chinese_answer_type_dict[t] - else: - return english_answer_type_dict[t] - exit(f"Error parsing answer type {answer_type}!") - - -def get_answer_type_text(answer_type, is_chinese, multiple_answer): - if ( - ("Need_human_evaluate" in answer_type) or ("Tuple" in answer_type) - ): # 'Tuple' has various meanings in different context, such as position or values of a series of variable, so it may lead to confusion to directly use 'tuple' in the prompt. - full_answer_text = "" - else: - if not multiple_answer: - answer_text = get_single_answer_type_text(answer_type, is_chinese) - if is_chinese: - full_answer_text = f"，答案类型为{answer_text}" - else: - full_answer_text = f"The answer of The problem should be {answer_text}. " - else: - if "," not in answer_type: # Same answer type for all answers - answer_text = get_single_answer_type_text(answer_type, is_chinese) - if is_chinese: - full_answer_text = f"，题目有多个答案，答案类型均为{answer_text}" - else: - full_answer_text = f"The problem has multiple answers, each of them should be {answer_text}. " - else: - answer_types = answer_type.split(",") - answer_types = [get_single_answer_type_text(t, is_chinese) for t in answer_types] - if len(set(answer_types)) == 1: - answer_text = answer_types[0] - if is_chinese: - full_answer_text = f"，题目有多个答案，答案类型均为{answer_text}" - else: - full_answer_text = f"The problem has multiple answers, each of them should be {answer_text}. " - else: - if is_chinese: - answer_text = "、".join(answer_types) - full_answer_text = f"，题目有多个答案，答案类型分别为{answer_text}" - else: - answer_text = ", ".join(answer_types) - full_answer_text = ( - f"The problem has multiple answers, with the answers in order being {answer_text}. " - ) - return full_answer_text - - -def create_record_to_sample(subset: str): - """Create a record_to_sample function for a specific subset.""" - is_theorem_proving = "TP" in subset - - def record_to_sample(record): - is_math = "Math" in record["subject"] - subject = "Math" if is_math else "Physics" - is_chinese = record["language"] == "Chinese" - unit = record.get("unit") - is_multiple_answer = record["is_multiple_answer"] - - if is_chinese: - subject_content = "数学" if is_math else "物理" - if is_theorem_proving: - instruction = f"以下是中国{subject_content}竞赛中的证明题。请根据题目的要求，运用逻辑推理及常用定理证明题目中的命题。证明过程中使用的变量和公式请使用LaTeX格式表示。" - else: - answer_type_text = get_answer_type_text( - record["answer_type"], is_chinese=True, multiple_answer=is_multiple_answer - ) - if is_multiple_answer: - multiple_answer_text = "\\boxed{用英文逗号连接的多个答案}" - else: - multiple_answer_text = "\\boxed{答案}" - unit_text = "" - if unit: - multiple_answer_text += "(单位)" - unit_text = "，注意答案的单位不要放在\\boxed{}中" - instruction = f'以下是中国{subject_content}竞赛中的解答题{answer_type_text}。请根据题目的要求和所提供的信息计算得出答案。解答过程和结果中使用的变量和公式请使用LaTeX格式表示。请在最后以"所以最终答案是{multiple_answer_text}。"显式给出结果{unit_text}。' - else: - if is_theorem_proving: - instruction = f"The following is a theorem proving problem from an International {subject} competition. Please use logical reasoning and common theorems to prove the proposition in the problem according to the given requirements. Please use LaTeX format to represent the variables and formulas used in the proof." - else: - if is_multiple_answer: - multiple_answer_text = "\\boxed{multiple answers connected with commas}" - else: - multiple_answer_text = "\\boxed{answer}" - unit_text = "" - if unit: - multiple_answer_text += "(unit)" - unit_text = ", note that the unit of the answer should not be included in \\boxed{}" - - answer_type_text = get_answer_type_text( - record["answer_type"], is_chinese=False, multiple_answer=is_multiple_answer - ) - - instruction = f'The following is an open-ended problem from an International {subject} competition. {answer_type_text}Please calculate the answer according to the given requirements and the information provided. Please use LaTeX format to represent the variables and formulas used in the solution process and results. Please end your solution with "So the final answer is {multiple_answer_text}." and give the result explicitly{unit_text}.' - - query = instruction + "\n" + record["question"] - - return Sample( - input=query, - target=record["final_answer"], - metadata={ - "language": record["language"], - "subject": record["subject"], - "answer_type": record["answer_type"], - "is_multiple_answer": record["is_multiple_answer"], - "unit": record.get("unit"), - }, - ) - - return record_to_sample - - -@scorer(metrics=[accuracy(), stderr()]) -def olympiad_bench_scorer(language: Language = Language.ENGLISH): - gold_extraction_target = (ExprExtractionConfig(), LatexExtractionConfig()) - pred_extraction_target = (ExprExtractionConfig(), LatexExtractionConfig()) - fallback_mode = "first_match" - extraction_mode = "first_match" - timeout_seconds = 5 - - gold_extraction_regexes = get_extraction_regexes_inspect(gold_extraction_target, language, len_choices=1) - pred_extraction_regexes = get_extraction_regexes_inspect(pred_extraction_target, language, len_choices=1) - - async def score(state: TaskState, target: Target): - extracted_predictions = extract_target_from_pred( - state.output.completion, pred_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds - ) - extracted_gold = extract_target_from_pred( - target.text, gold_extraction_regexes, fallback_mode, extraction_mode, timeout_seconds - ) - return Score( - value="C" if extracted_predictions == extracted_gold else "I", - explanation=state.output.completion, - answer=str(extracted_predictions), - ) - - return score - - -# Very specific task where there are no precise outputs but instead we test if the format obeys rules -def olympiad_bench_prompt(line, task_name: str = None): - is_math = "Math" in line["subject"] - subject = "Math" if is_math else "Physics" - is_chinese = line["language"] == "Chinese" - is_theorem_proving = "TP" in task_name - unit = line["unit"] - is_multiple_answer = line["is_multiple_answer"] - - if is_chinese: - subject_content = "数学" if is_math else "物理" - if is_theorem_proving: - instruction = f"以下是中国{subject_content}竞赛中的证明题。请根据题目的要求，运用逻辑推理及常用定理证明题目中的命题。证明过程中使用的变量和公式请使用LaTeX格式表示。" - else: - answer_type_text = get_answer_type_text( - line["answer_type"], is_chinese=True, multiple_answer=is_multiple_answer - ) - if is_multiple_answer: - multiple_answer_text = "\\boxed{用英文逗号连接的多个答案}" - else: - multiple_answer_text = "\\boxed{答案}" - unit_text = "" - if unit: - multiple_answer_text += "(单位)" - unit_text = "，注意答案的单位不要放在\\boxed{}中" - instruction = f"以下是中国{subject_content}竞赛中的解答题{answer_type_text}。请根据题目的要求和所提供的信息计算得出答案。解答过程和结果中使用的变量和公式请使用LaTeXæ ¼式表示。请在最后以“所以最终答案是{multiple_answer_text}。”显式给出结果{unit_text}。" - else: - if is_theorem_proving: - instruction = f"The following is a theorem proving problem from an International {subject} competition. Please use logical reasoning and common theorems to prove the proposition in the problem according to the given requirements. Please use LaTeX format to represent the variables and formulas used in the proof." - else: - if is_multiple_answer: - multiple_answer_text = "\\boxed{multiple answers connected with commas}" - else: - multiple_answer_text = "\\boxed{answer}" - unit_text = "" - if unit: - multiple_answer_text += "(unit)" - unit_text = ", note that the unit of the answer should not be included in \\boxed{}" - - answer_type_text = get_answer_type_text( - line["answer_type"], is_chinese=False, multiple_answer=is_multiple_answer - ) - - instruction = f'The following is an open-ended problem from an International {subject} competition. {answer_type_text}Please calculate the answer according to the given requirements and the information provided. Please use LaTeX format to represent the variables and formulas used in the solution process and results. Please end your solution with "So the final answer is {multiple_answer_text}." and give the result explicitly{unit_text}.' - - choice = line["final_answer"] - - return Doc( - task_name=task_name, - query=instruction + "\n" + line["question"], - choices=[choice], - gold_index=0, - instruction=instruction, - specific={}, - ) - - -# * OE: Open-ended questions -# * TP: Theorem proof problems -# * MM: Multimodal -# * TO: Text-only -# * physics: Physics problems -# * maths: Math problems -# * en: English -# * zh: Chinese -# * COMP: Competition problems -# * CEE: Chinese College Entrance Exam problems - -question_type = ["OE"] # TP # TP cannot be replicated -multimodality = ["TO"] # MM # We do not allow multimodal tasks -subject = ["physics", "maths"] -language = ["en", "zh"] -source = ["COMP", "CEE"] - -olympiad_bench_subsets = [] - -for qt in question_type: - for mm in multimodality: - for sub in subject: - for lang in language: - for src in source: - olympiad_bench_subsets.append(f"{qt}_{mm}_{sub}_{lang}_{src}") - -available_subsets = [ - "OE_MM_maths_en_COMP", - "OE_MM_maths_zh_CEE", - "OE_MM_maths_zh_COMP", - "OE_MM_physics_en_COMP", - "OE_MM_physics_zh_CEE", - "OE_TO_maths_en_COMP", - "OE_TO_maths_zh_CEE", - "OE_TO_maths_zh_COMP", - "OE_TO_physics_en_COMP", - "OE_TO_physics_zh_CEE", - "TP_MM_maths_en_COMP", - "TP_MM_maths_zh_CEE", - "TP_MM_maths_zh_COMP", - "TP_MM_physics_en_COMP", - "TP_TO_maths_en_COMP", - "TP_TO_maths_zh_CEE", - "TP_TO_maths_zh_COMP", - "TP_TO_physics_en_COMP", -] - -olympiad_bench_subsets = set(olympiad_bench_subsets).intersection(available_subsets) - -task_configs = [] - -for subset in olympiad_bench_subsets: - # Determine language from subset name (en or zh) - is_chinese = "_zh_" in subset - language = Language.CHINESE if is_chinese else Language.ENGLISH - - # We create the task config - task_configs.append( - LightevalTaskConfig( - name="olympiad_bench:" + subset, - prompt_function=olympiad_bench_prompt, - hf_repo="Hothan/OlympiadBench", - hf_subset=subset, - metrics=[Metrics.exact_match], - sample_fields=create_record_to_sample(subset), - solver=[generate(cache=True)], - scorer=olympiad_bench_scorer(language=language), - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split="train", - few_shots_select="random_sampling", - generation_size=2048, - stop_sequence=[], # no stop sequence, will use eot token - version=1, - ) - ) - - -TASKS_TABLE = task_configs diff --git a/src/lighteval/tasks/tasks/openbookqa.py b/src/lighteval/tasks/tasks/openbookqa.py deleted file mode 100644 index 62c378426..000000000 --- a/src/lighteval/tasks/tasks/openbookqa.py +++ /dev/null @@ -1,67 +0,0 @@ -""" -name: -Openbookqa - -dataset: -allenai/openbookqa - -abstract: -OpenBookQA is a question-answering dataset modeled after open-book exams for -assessing human understanding of a subject. It contains multiple-choice -questions that require combining facts from a given open book with broad common -knowledge. The task tests language models' ability to leverage provided -information and apply common sense reasoning. - -languages: -english - -tags: -multiple-choice, qa - -paper: -https://arxiv.org/abs/1809.02789 -""" - -from string import ascii_uppercase - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def openbookqa_prompt(line, task_name: str = None): - query = "The following are multiple choice questions (with answers) about common sense.\n" - query += f"Question: {line['question_stem']}\n" - query += "".join([f"{key}. {choice}\n" for key, choice in zip(ascii_uppercase, line["choices"]["text"])]) - query += "Answer: " - - gold_ix = ["A", "B", "C", "D", "E"].index(line["answerKey"].strip()) - return Doc( - task_name=task_name, - query=query, - choices=list(ascii_uppercase[: len(line["choices"]["text"])]), - gold_index=gold_ix, - instruction="The following are multiple choice questions (with answers) about common sense.\n", - ) - - -openbookqa = LightevalTaskConfig( - name="openbookqa", - prompt_function=openbookqa_prompt, - hf_repo="allenai/openbookqa", - hf_subset="main", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - openbookqa, -] diff --git a/src/lighteval/tasks/tasks/piqa.py b/src/lighteval/tasks/tasks/piqa.py deleted file mode 100644 index 4aa20719c..000000000 --- a/src/lighteval/tasks/tasks/piqa.py +++ /dev/null @@ -1,66 +0,0 @@ -""" -name: -Piqa - -dataset: -ybisk/piqa - -abstract: -PIQA is a benchmark for testing physical commonsense reasoning. It contains -questions requiring this kind of physical commonsense reasoning. - -languages: -english - -tags: -commonsense, multiple-choice, qa - -paper: -https://arxiv.org/abs/1911.11641 -""" - -from string import ascii_uppercase - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def piqa_prompt(line, task_name: str = None): - letters = list(ascii_uppercase)[:2] - query = "The following are multiple choice questions (with answers) about common sense.\n" - query += f"Question: {line['goal']}\n" - query += "".join([f"{key}. {choice}\n" for key, choice in zip(letters, [line["sol1"], line["sol2"]])]) - query += "Answer: " - - gold_ix = int(line["label"]) - is_few_shots = line.get("__few_shots", False) - return Doc( - task_name=task_name, - query=query, - choices=letters if not is_few_shots else [line["sol1"], line["sol2"]], - gold_index=gold_ix, - instruction="The following are multiple choice questions (with answers) about common sense.\n", - ) - - -piqa = LightevalTaskConfig( - name="piqa", - prompt_function=piqa_prompt, - hf_repo="ybisk/piqa", - hf_subset="plain_text", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - piqa, -] diff --git a/src/lighteval/tasks/tasks/prost.py b/src/lighteval/tasks/tasks/prost.py deleted file mode 100644 index 97a3c7178..000000000 --- a/src/lighteval/tasks/tasks/prost.py +++ /dev/null @@ -1,56 +0,0 @@ -""" -name: -Prost - -dataset: -lighteval/prost - -abstract: -PROST is a benchmark for testing physical reasoning about objects through space -and time. It includes 18,736 multiple-choice questions covering 10 core physics -concepts, designed to probe models in zero-shot settings. Results show that even -large pretrained models struggle with physical reasoning and are sensitive to -question phrasing, underscoring their limited real-world understanding. - -languages: -english - -tags: -reasoning, qa, physical-commonsense - -paper: -https://arxiv.org/abs/2106.03634 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def prost_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=line["question"], - choices=[f" {c}" for c in line["choices"]], - gold_index=int(line["label"]) if isinstance(line["label"], int) else int(line["label"]), - ) - - -prost = LightevalTaskConfig( - name="prost", - prompt_function=prost_prompt, - hf_repo="lighteval/prost", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - prost, -] diff --git a/src/lighteval/tasks/tasks/pubmedqa.py b/src/lighteval/tasks/tasks/pubmedqa.py deleted file mode 100644 index 326789bb6..000000000 --- a/src/lighteval/tasks/tasks/pubmedqa.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -name: -Pubmedqa - -dataset: -pubmed_qa - -abstract: -PubMedQA is a dataset for biomedical research question answering. - -languages: -english - -tags: -biomedical, health, medical, qa - -paper: -https://pubmedqa.github.io/ -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def pubmed_qa_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"{line['QUESTION']}\n{line['CONTEXTS']}\nAnswer: ", - choices=[line["final_decision"]], - gold_index=0, - ) - - -pubmedqa = LightevalTaskConfig( - name="pubmedqa", - prompt_function=pubmed_qa_prompt, - hf_repo="pubmed_qa", - hf_subset="pqa_labeled", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - pubmedqa, -] diff --git a/src/lighteval/tasks/tasks/qa4mre.py b/src/lighteval/tasks/tasks/qa4mre.py deleted file mode 100644 index c0a4cab3e..000000000 --- a/src/lighteval/tasks/tasks/qa4mre.py +++ /dev/null @@ -1,96 +0,0 @@ -""" -name: -Qa4Mre - -dataset: -qa4mre - -abstract: -QA4MRE is a machine reading comprehension benchmark from the CLEF 2011-2013 -challenges. It evaluates systems' ability to answer questions requiring deep -understanding of short texts, supported by external background knowledge. -Covering tasks like modality, negation, biomedical reading, and entrance exams, -QA4MRE tests reasoning beyond surface-level text matching. - -languages: -english - -tags: -biomedical, health, qa - -paper: -https://link.springer.com/chapter/10.1007/978-3-642-40802-1_29 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def qa4mre_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"{line['question']}", - choices=[f" {c}" for c in line["choices"]], - gold_index=line["label"], - ) - - -qa4mre_2011 = LightevalTaskConfig( - name="qa4mre:2011", - prompt_function=qa4mre_prompt, - hf_repo="qa4mre", - hf_subset="2011.main.EN", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=["\n"], - version=0, -) - - -qa4mre_2012 = LightevalTaskConfig( - name="qa4mre:2012", - prompt_function=qa4mre_prompt, - hf_repo="qa4mre", - hf_subset="2012.main.EN", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=["\n"], - version=0, -) - - -qa4mre_2013 = LightevalTaskConfig( - name="qa4mre:2013", - prompt_function=qa4mre_prompt, - hf_repo="qa4mre", - hf_subset="2013.main.EN", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[ - Metrics.loglikelihood_acc, - ], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - qa4mre_2011, - qa4mre_2012, - qa4mre_2013, -] diff --git a/src/lighteval/tasks/tasks/qasper.py b/src/lighteval/tasks/tasks/qasper.py deleted file mode 100644 index 59bd5ee90..000000000 --- a/src/lighteval/tasks/tasks/qasper.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -name: -Qasper - -dataset: -allenai/qasper - -abstract: -QASPER is a dataset for question answering on scientific research papers. It -consists of 5,049 questions over 1,585 Natural Language Processing papers. Each -question is written by an NLP practitioner who read only the title and abstract -of the corresponding paper, and the question seeks information present in the -full text. The questions are then answered by a separate set of NLP -practitioners who also provide supporting evidence to answers. - -languages: -english - -tags: -qa, scientific - -paper: -https://arxiv.org/abs/2105.03011 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def qasper_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"Title: {line['title']}\n\nPassage: {line['passage']}\n\n Question: {line['question']}\nAnswer: ", - gold_index=0, - choices=[line["gold"]], - ) - - -qasper = LightevalTaskConfig( - name="qasper", - prompt_function=qasper_prompt, - hf_repo="allenai/qasper", - hf_subset="qasper", - hf_avail_splits=["train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.f1_score], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - qasper, -] diff --git a/src/lighteval/tasks/tasks/quac.py b/src/lighteval/tasks/tasks/quac.py deleted file mode 100644 index 67268831d..000000000 --- a/src/lighteval/tasks/tasks/quac.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -name: -Quac - -dataset: -lighteval/quac_helm - -abstract: -The QuAC benchmark for question answering in the context of dialogues. - -languages: -english - -tags: -dialog, qa - -paper: -https://aclanthology.org/D18-1241/ -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def quac_prompt(line, task_name: str = None): - references = [ref for ref in line["references"] if ref is not None and ref != ""] - return Doc( - task_name=task_name, - query=f"{line['prompt']}\nAnswer:", - choices=references, - gold_index=list(range(len(references))), - ) - - -quac = LightevalTaskConfig( - name="quac", - prompt_function=quac_prompt, - hf_repo="lighteval/quac_helm", - hf_subset="default", - hf_avail_splits=["train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=100, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - quac, -] diff --git a/src/lighteval/tasks/tasks/race_high.py b/src/lighteval/tasks/tasks/race_high.py deleted file mode 100644 index 37eddaf2d..000000000 --- a/src/lighteval/tasks/tasks/race_high.py +++ /dev/null @@ -1,68 +0,0 @@ -""" -name: -Race High - -dataset: -EleutherAI/race - -abstract: -RACE is a large-scale reading comprehension dataset with more than 28,000 -passages and nearly 100,000 questions. The dataset is collected from English -examinations in China, which are designed for middle school and high school -students. The dataset can be served as the training and test sets for machine -comprehension. - -languages: -english - -tags: -multiple-choice, reading-comprehension - -paper: -https://aclanthology.org/D17-1082/ -""" - -import ast - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def race_prompt(line, task_name: str = None): - line["problems"] = ast.literal_eval(line["problems"]) - text = f"Article: {line['article']}\n\n" - for problem in line["problems"][:-1]: - index = ["A", "B", "C", "D", "E"].index(problem["answer"]) - if problem["question"][-6:] == " _ .": - text += f"{problem['question'][-5:]}{problem['options'][index]}\n" - else: - text += f"Question: {problem['question']}\n" - text += f"Answer: {problem['options'][index]}\n" - text += line["problems"][-1]["question"] - return Doc( - task_name=task_name, - query=text, - choices=[f" {o}" for o in line["problems"][-1]["options"]], - gold_index=["A", "B", "C", "D", "E"].index(line["problems"][-1]["answer"]), - ) - - -race_high = LightevalTaskConfig( - name="race:high", - prompt_function=race_prompt, - hf_repo="EleutherAI/race", - hf_subset="high", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - race_high, -] diff --git a/src/lighteval/tasks/tasks/raft.py b/src/lighteval/tasks/tasks/raft.py deleted file mode 100644 index 73c03c0e2..000000000 --- a/src/lighteval/tasks/tasks/raft.py +++ /dev/null @@ -1,299 +0,0 @@ -""" -name: -Raft - -dataset: -ought/raft - -abstract: -The Real-world annotated few-shot (RAFT) meta-benchmark of 11 real-world text -classification tasks. - -languages: -english - -tags: -classification, reasoning - -paper: -https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/hash/ca46c1b9512a7a8315fa3c5a946e8265-Abstract-round2.html -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def raft_prompt(line, query_keys, instruction, task_name: str = None): - query = instruction - query += "\n".join([f"{key}: {line[key]}" for key in query_keys]) - query += "\nLabel:" - return Doc(task_name=task_name, query=query, gold_index=0, choices=[str(line["Label"])], instruction=instruction) - - -def raft_ade_corpus_v2_prompt(line, task_name: str = None): - instruction = "Label the sentence based on whether it is related to an adverse drug effect (ADE). Details are described below:\nDrugs: Names of drugs and chemicals that include brand names, trivial names, abbreviations and systematic names were annotated. Mentions of drugs or chemicals should strictly be in a therapeutic context. This category does not include the names of metabolites, reaction byproducts, or hospital chemicals (e.g. surgical equipment disinfectants).\nAdverse effect: Mentions of adverse effects include signs, symptoms, diseases, disorders, acquired abnormalities, deficiencies, organ damage or death that strictly occur as a consequence of drug intake.\nPossible labels:\n1. ADE-related\n2. not ADE-related" - query_keys = ["Sentence"] - return raft_prompt(line, query_keys, instruction, task_name) - - -def raft_banking_77_prompt(line, task_name: str = None): - instruction = "The following is a banking customer service query. Classify the query into one of the 77 categories available.\nPossible labels:\n1. Refund_not_showing_up\n2. activate_my_card\n3. age_limit\n4. apple_pay_or_google_pay\n5. atm_support\n6. automatic_top_up\n7. balance_not_updated_after_bank_transfer\n8. balance_not_updated_after_cheque_or_cash_deposit\n9. beneficiary_not_allowed\n10. cancel_transfer\n11. card_about_to_expire\n12. card_acceptance\n13. card_arrival\n14. card_delivery_estimate\n15. card_linking\n16. card_not_working\n17. card_payment_fee_charged\n18. card_payment_not_recognised\n19. card_payment_wrong_exchange_rate\n20. card_swallowed\n21. cash_withdrawal_charge\n22. cash_withdrawal_not_recognised\n23. change_pin\n24. compromised_card\n25. contactless_not_working\n26. country_support\n27. declined_card_payment\n28. declined_cash_withdrawal\n29. declined_transfer\n30. direct_debit_payment_not_recognised\n31. disposable_card_limits\n32. edit_personal_details\n33. exchange_charge\n34. exchange_rate\n35. exchange_via_app\n36. extra_charge_on_statement\n37. failed_transfer\n38. fiat_currency_support\n39. get_disposable_virtual_card\n40. get_physical_card\n41. getting_spare_card\n42. getting_virtual_card\n43. lost_or_stolen_card\n44. lost_or_stolen_phone\n45. order_physical_card\n46. passcode_forgotten\n47. pending_card_payment\n48. pending_cash_withdrawal\n49. pending_top_up\n50. pending_transfer\n51. pin_blocked\n52. receiving_money\n53. request_refund\n54. reverted_card_payment?\n55. supported_cards_and_currencies\n56. terminate_account\n57. top_up_by_bank_transfer_charge\n58. top_up_by_card_charge\n59. top_up_by_cash_or_cheque\n60. top_up_failed\n61. top_up_limits\n62. top_up_reverted\n63. topping_up_by_card\n64. transaction_charged_twice\n65. transfer_fee_charged\n66. transfer_into_account\n67. transfer_not_received_by_recipient\n68. transfer_timing\n69. unable_to_verify_identity\n70. verify_my_identity\n71. verify_source_of_funds\n72. verify_top_up\n73. virtual_card_not_working\n74. visa_or_mastercard\n75. why_verify_identity\n76. wrong_amount_of_cash_received\n77. wrong_exchange_rate_for_cash_withdrawal" - query_keys = ["Query"] - return raft_prompt(line, query_keys, instruction, task_name) - - -def raft_neurips_impact_statement_risks_prompt(line, task_name: str = None): - instruction = "Label the impact statement based on whether it mentions a harmful application of the research done in the paper. Make sure the statement is sufficient to conclude there are harmful applications of the research being done, not a past risk that this research is solving.\nPossible labels:\n1. doesn't mention a harmful application\n2. mentions a harmful application" - query_keys = ["Impact statement", "Paper title"] - return raft_prompt(line, query_keys, instruction, task_name) - - -def raft_one_stop_english_prompt(line, task_name: str = None): - instruction = "The following is an article sourced from The Guardian newspaper, and rewritten by teachers to suit three levels of adult English as Second Language (ESL) learners: elementary, intermediate, and advanced. Predict the level of the article.\nPossible labels:\n1. advanced\n2. elementary\n3. intermediate" - query_keys = ["Article"] - return raft_prompt(line, query_keys, instruction, task_name) - - -def raft_overruling_prompt(line, task_name: str = None): - instruction = "In law, an overruling sentence is a statement that nullifies a previous case decision as a precedent, by a constitutionally valid statute or a decision by the same or higher ranking court which establishes a different rule on the point of law involved. Label the sentence based on whether it is overruling or not.\nPossible labels:\n1. not overruling\n2. overruling" - query_keys = ["Sentence"] - return raft_prompt(line, query_keys, instruction, task_name) - - -def raft_semiconductor_org_types_prompt(line, task_name: str = None): - instruction = 'The dataset is a list of institutions that have contributed papers to semiconductor conferences in the last 25 years, as catalogued by IEEE and sampled randomly. The goal is to classify the institutions into one of three categories: "university", "company" or "research institute".\nPossible labels:\n1. company\n2. research institute\n3. university' - query_keys = ["Organization name", "Paper title"] - return raft_prompt(line, query_keys, instruction, task_name) - - -def raft_systematic_review_inclusion_prompt(line, task_name: str = None): - instruction = "Identify whether this paper should be included in a meta-review which includes the findings of systematic reviews on interventions designed to promote charitable donations.\nIncluded reviews should describe monetary charitable donations, assess any population of participants in any context, and be peer reviewed and written in English.\nThey should not report new data, be non-systematic reviews, consider cause-related marketing or other kinds of prosocial behaviour.\nPossible labels:\n1. included\n2. not included" - query_keys = ["Title", "Abstract", "Journal"] - return raft_prompt(line, query_keys, instruction, task_name) - - -def raft_tai_safety_research_prompt(line, task_name: str = None): - instruction = 'Transformative AI (TAI) is defined as AI that precipitates a transition comparable to (or more significant than) the agricultural or industrial revolution. Label a paper as "TAI safety research" if:\n1. The contents of the paper are directly motivated by, and substantively inform, the challenge of ensuring good outcomes for TAI,\n2. There is substantive content on AI safety, not just AI capabilities,\n3. The intended audience is the community of researchers,\n4. It meets a subjective threshold of seriousness/quality,\n5. Peer review is not required.\nPossible labels:\n1. TAI safety research\n2. not TAI safety research' - query_keys = ["Title", "Abstract Note", "Publication Title", "Item Type", "Publication Year"] - return raft_prompt(line, query_keys, instruction, task_name) - - -def raft_terms_of_service_prompt(line, task_name: str = None): - instruction = "Label the sentence from a Terms of Service based on whether it is potentially unfair. If it seems clearly unfair, mark it as potentially unfair.\nAccording to art. 3 of the Directive 93/13 on Unfair Terms in Consumer Contracts, a contractual term is unfair if: 1) it has not been individually negotiated; and 2) contrary to the requirement of good faith, it causes a significant imbalance in the parties rights and obligations, to the detriment of the consumer.\nDetails on types of potentially unfair clauses are found below:\nThe jurisdiction clause stipulates what courts will have the competence to adjudicate disputes under the contract. Jurisdiction clauses giving consumers a right to bring disputes in their place of residence were marked as clearly fair, whereas clauses stating that any judicial proceeding takes a residence away were marked as clearly unfair.\nThe choice of law clause specifies what law will govern the contract, meaning also what law will be applied in potential adjudication of a dispute arising under the contract. Clauses defining the applicable law as the law of the consumer's country of residence were marked as clearly fair. In every other case, the choice of law clause was considered as potentially unfair.\nThe limitation of liability clause stipulates that the duty to pay damages is limited or excluded, for certain kind of losses, under certain conditions. Clauses that explicitly affirm non-excludable providers' liabilities were marked as clearly fair. Clauses that reduce, limit, or exclude the liability of the service provider were marked as potentially unfair when concerning broad categories of losses or causes of them.\nThe unilateral change clause specifies the conditions under which the service provider could amend and modify the terms of service and/or the service itself. Such clause was always considered as potentially unfair.\nThe unilateral termination clause gives provider the right to suspend and/or terminate the service and/or the contract, and sometimes details the circumstances under which the provider claims to have a right to do so.\nThe contract by using clause stipulates that the consumer is bound by the terms of use of a specific service, simply by using the service, without even being required to mark that he or she has read and accepted them. We always marked such clauses as potentially unfair.\nThe content removal gives the provider a right to modify/delete user's content, including in-app purchases, and sometimes specifies the conditions under which the service provider may do so.\nThe arbitration clause requires or allows the parties to resolve their disputes through an arbitration process, before the case could go to court. Clauses stipulating that the arbitration should take place in a state other then the state of consumer's residence or be based on arbiter's discretion were marked as clearly unfair. Clauses defining arbitration as fully optional were marked as clearly fair.\nPossible labels:\n1. not potentially unfair\n2. potentially unfair" - query_keys = ["Sentence"] - return raft_prompt(line, query_keys, instruction, task_name) - - -def raft_tweet_eval_hate_prompt(line, task_name: str = None): - instruction = "Label whether the following tweet contains hate speech against either immigrants or women. Hate Speech (HS) is commonly defined as any communication that disparages a person or a group on the basis of some characteristic such as race, color, ethnicity, gender, sexual orientation, nationality, religion, or other characteristics.\nPossible labels:\n1. hate speech\n2. not hate speech" - query_keys = ["Tweet"] - return raft_prompt(line, query_keys, instruction, task_name) - - -def raft_twitter_complaints_prompt(line, task_name: str = None): - instruction = "A complaint presents a state of affairs which breaches the writer\u2019s favorable expectation. Label the tweet text based on whether it contains a complaint.\nPossible labels:\n1. complaint\n2. no complaint" - query_keys = ["Tweet text"] - return raft_prompt(line, query_keys, instruction, task_name) - - -raft_ade_corpus_v2 = LightevalTaskConfig( - name="raft:ade_corpus_v2", - prompt_function=raft_ade_corpus_v2_prompt, - hf_repo="ought/raft", - hf_subset="ade_corpus_v2", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=30, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -raft_banking_77 = LightevalTaskConfig( - name="raft:banking_77", - prompt_function=raft_banking_77_prompt, - hf_repo="ought/raft", - hf_subset="banking_77", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=30, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -raft_neurips_impact_statement_risks = LightevalTaskConfig( - name="raft:neurips_impact_statement_risks", - prompt_function=raft_neurips_impact_statement_risks_prompt, - hf_repo="ought/raft", - hf_subset="neurips_impact_statement_risks", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=30, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -raft_one_stop_english = LightevalTaskConfig( - name="raft:one_stop_english", - prompt_function=raft_one_stop_english_prompt, - hf_repo="ought/raft", - hf_subset="one_stop_english", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=30, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -raft_overruling = LightevalTaskConfig( - name="raft:overruling", - prompt_function=raft_overruling_prompt, - hf_repo="ought/raft", - hf_subset="overruling", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=30, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -raft_semiconductor_org_types = LightevalTaskConfig( - name="raft:semiconductor_org_types", - prompt_function=raft_semiconductor_org_types_prompt, - hf_repo="ought/raft", - hf_subset="semiconductor_org_types", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=30, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -raft_systematic_review_inclusion = LightevalTaskConfig( - name="raft:systematic_review_inclusion", - prompt_function=raft_systematic_review_inclusion_prompt, - hf_repo="ought/raft", - hf_subset="systematic_review_inclusion", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=30, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -raft_tai_safety_research = LightevalTaskConfig( - name="raft:tai_safety_research", - prompt_function=raft_tai_safety_research_prompt, - hf_repo="ought/raft", - hf_subset="tai_safety_research", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=30, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -raft_terms_of_service = LightevalTaskConfig( - name="raft:terms_of_service", - prompt_function=raft_terms_of_service_prompt, - hf_repo="ought/raft", - hf_subset="terms_of_service", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=30, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -raft_tweet_eval_hate = LightevalTaskConfig( - name="raft:tweet_eval_hate", - prompt_function=raft_tweet_eval_hate_prompt, - hf_repo="ought/raft", - hf_subset="tweet_eval_hate", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=30, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -raft_twitter_complaints = LightevalTaskConfig( - name="raft:twitter_complaints", - prompt_function=raft_twitter_complaints_prompt, - hf_repo="ought/raft", - hf_subset="twitter_complaints", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=30, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - raft_ade_corpus_v2, - raft_banking_77, - raft_neurips_impact_statement_risks, - raft_one_stop_english, - raft_overruling, - raft_semiconductor_org_types, - raft_systematic_review_inclusion, - raft_tai_safety_research, - raft_terms_of_service, - raft_tweet_eval_hate, - raft_twitter_complaints, -] diff --git a/src/lighteval/tasks/tasks/real_toxicity_prompts.py b/src/lighteval/tasks/tasks/real_toxicity_prompts.py deleted file mode 100644 index 7d6f2faf0..000000000 --- a/src/lighteval/tasks/tasks/real_toxicity_prompts.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -name: -Real Toxicity Prompts - -dataset: -allenai/real-toxicity-prompts - -abstract: -The RealToxicityPrompts dataset for measuring toxicity in prompted model generations - -languages: -english - -tags: -generation, safety - -paper: -https://aclanthology.org/2020.findings-emnlp.301/ -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def real_toxicity_prompts_prompt(line, task_name: str = None): - # Some variants store text under 'prompt' -> 'text'; handle both flat and nested - text = ( - line["prompt"]["text"] - if isinstance(line.get("prompt"), dict) and "text" in line["prompt"] - else line.get("text", "") - ) - return Doc(task_name=task_name, query=text, choices=None, gold_index=None) - - -real_toxicity_prompts = LightevalTaskConfig( - name="real_toxicity_prompts", - prompt_function=real_toxicity_prompts_prompt, - hf_repo="allenai/real-toxicity-prompts", - hf_subset="default", - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - real_toxicity_prompts, -] diff --git a/src/lighteval/tasks/tasks/sacrebleu.py b/src/lighteval/tasks/tasks/sacrebleu.py deleted file mode 100644 index 8b2aa0bc6..000000000 --- a/src/lighteval/tasks/tasks/sacrebleu.py +++ /dev/null @@ -1,2785 +0,0 @@ -""" -name: -Sacrebleu - -dataset: -lighteval/sacrebleu_manual, wmt14, wmt16 - -abstract: -tasks from sacrebleu - -languages: -english, german, french, japanese, korean, chinese, arabic - -tags: -translation - -paper: -https://github.com/mjpost/sacrebleu -""" - -import ast - -import pycountry - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc -from lighteval.utils.utils import as_list - - -def __wmt_prompt(line, alphabetical, task_name: str = None): - def language(code): - # key is alpha_2 or alpha_3 depending on the code length - language_tuple = pycountry.languages.get(**{f"alpha_{len(code)}": code}) - return language_tuple.name - - # It would be better to just reupload the file tbh - if isinstance(line["translation"], str): - line["translation"] = ast.literal_eval(line["translation"]) - for k, v in line["translation"].items(): - line["translation"][k] = as_list(v)[0] - - l_in, l_out = sorted(line["translation"].keys(), reverse=not alphabetical) - - return Doc( - task_name=task_name, - query=f"{language(l_in)} phrase: " + line["translation"][l_in].rstrip() + f"\n{language(l_out)} phrase:", - gold_index=0, - choices=[line["translation"][l_out].rstrip()], - instruction=f"Translate {language(l_in)} to {language(l_out)}, do not explain, only output the translation.", - ) - - -def wmt_alphabetical_prompt(line, task_name: str = None): - return __wmt_prompt(line, True, task_name) - - -def wmt_reverse_alphabetical_prompt(line, task_name: str = None): - return __wmt_prompt(line, False, task_name) - - -iwslt17_ar_en = LightevalTaskConfig( - name="iwslt17:ar-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="iwslt17_ar-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -iwslt17_de_en = LightevalTaskConfig( - name="iwslt17:de-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="iwslt17_de-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -iwslt17_en_ar = LightevalTaskConfig( - name="iwslt17:en-ar", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="iwslt17_ar-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -iwslt17_en_de = LightevalTaskConfig( - name="iwslt17:en-de", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="iwslt17_en-de", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -iwslt17_en_fr = LightevalTaskConfig( - name="iwslt17:en-fr", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="iwslt17_en-fr", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -iwslt17_en_ja = LightevalTaskConfig( - name="iwslt17:en-ja", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="iwslt17_en-ja", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -iwslt17_en_ko = LightevalTaskConfig( - name="iwslt17:en-ko", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="iwslt17_en-ko", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -iwslt17_en_zh = LightevalTaskConfig( - name="iwslt17:en-zh", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="iwslt17_en-zh", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -iwslt17_fr_en = LightevalTaskConfig( - name="iwslt17:fr-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="iwslt17_fr-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -iwslt17_ja_en = LightevalTaskConfig( - name="iwslt17:ja-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="iwslt17_ja-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -iwslt17_ko_en = LightevalTaskConfig( - name="iwslt17:ko-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="iwslt17_ko-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -iwslt17_zh_en = LightevalTaskConfig( - name="iwslt17:zh-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="iwslt17_zh-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -mtnt2019_en_fr = LightevalTaskConfig( - name="mtnt2019:en-fr", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="mtnt2019_en-fr", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=200, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -mtnt2019_en_ja = LightevalTaskConfig( - name="mtnt2019:en-ja", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="mtnt2019_en-ja", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=200, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -mtnt2019_fr_en = LightevalTaskConfig( - name="mtnt2019:fr-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="mtnt2019_fr-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=200, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -mtnt2019_ja_en = LightevalTaskConfig( - name="mtnt2019:ja-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="mtnt2019_ja-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=200, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt08_cs_en = LightevalTaskConfig( - name="wmt08:cs-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt08_cs-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt08_de_en = LightevalTaskConfig( - name="wmt08:de-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt08_de-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt08_en_cs = LightevalTaskConfig( - name="wmt08:en-cs", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt08_en-cs", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt08_en_de = LightevalTaskConfig( - name="wmt08:en-de", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt08_en-de", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt08_en_es = LightevalTaskConfig( - name="wmt08:en-es", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt08_en-es", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt08_en_fr = LightevalTaskConfig( - name="wmt08:en-fr", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt08_en-fr", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt08_en_hu = LightevalTaskConfig( - name="wmt08:en-hu", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt08_en-hu", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt08_es_en = LightevalTaskConfig( - name="wmt08:es-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt08_es-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt08_fr_en = LightevalTaskConfig( - name="wmt08:fr-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt08_fr-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt08_hu_en = LightevalTaskConfig( - name="wmt08:hu-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt08_hu-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt09_cs_en = LightevalTaskConfig( - name="wmt09:cs-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt09_cs-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt09_de_en = LightevalTaskConfig( - name="wmt09:de-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt09_de-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt09_en_cs = LightevalTaskConfig( - name="wmt09:en-cs", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt09_en-cs", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt09_en_de = LightevalTaskConfig( - name="wmt09:en-de", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt09_en-de", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt09_en_es = LightevalTaskConfig( - name="wmt09:en-es", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt09_en-es", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt09_en_fr = LightevalTaskConfig( - name="wmt09:en-fr", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt09_en-fr", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt09_en_hu = LightevalTaskConfig( - name="wmt09:en-hu", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt09_en-hu", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt09_en_it = LightevalTaskConfig( - name="wmt09:en-it", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt09_en-it", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt09_es_en = LightevalTaskConfig( - name="wmt09:es-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt09_es-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt09_fr_en = LightevalTaskConfig( - name="wmt09:fr-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt09_fr-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt09_hu_en = LightevalTaskConfig( - name="wmt09:hu-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt09_hu-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt09_it_en = LightevalTaskConfig( - name="wmt09:it-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt09_it-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt10_cs_en = LightevalTaskConfig( - name="wmt10:cs-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt10_cs-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt10_de_en = LightevalTaskConfig( - name="wmt10:de-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt10_de-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt10_en_cs = LightevalTaskConfig( - name="wmt10:en-cs", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt10_en-cs", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt10_en_de = LightevalTaskConfig( - name="wmt10:en-de", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt10_en-de", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt10_en_es = LightevalTaskConfig( - name="wmt10:en-es", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt10_en-es", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt10_en_fr = LightevalTaskConfig( - name="wmt10:en-fr", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt10_en-fr", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt10_es_en = LightevalTaskConfig( - name="wmt10:es-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt10_es-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt10_fr_en = LightevalTaskConfig( - name="wmt10:fr-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt10_fr-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt11_cs_en = LightevalTaskConfig( - name="wmt11:cs-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt11_cs-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt11_de_en = LightevalTaskConfig( - name="wmt11:de-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt11_de-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt11_en_cs = LightevalTaskConfig( - name="wmt11:en-cs", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt11_en-cs", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt11_en_de = LightevalTaskConfig( - name="wmt11:en-de", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt11_en-de", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt11_en_es = LightevalTaskConfig( - name="wmt11:en-es", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt11_en-es", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt11_en_fr = LightevalTaskConfig( - name="wmt11:en-fr", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt11_en-fr", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt11_es_en = LightevalTaskConfig( - name="wmt11:es-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt11_es-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt11_fr_en = LightevalTaskConfig( - name="wmt11:fr-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt11_fr-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt12_cs_en = LightevalTaskConfig( - name="wmt12:cs-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt12_cs-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt12_de_en = LightevalTaskConfig( - name="wmt12:de-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt12_de-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt12_en_cs = LightevalTaskConfig( - name="wmt12:en-cs", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt12_en-cs", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt12_en_de = LightevalTaskConfig( - name="wmt12:en-de", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt12_en-de", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt12_en_es = LightevalTaskConfig( - name="wmt12:en-es", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt12_en-es", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt12_en_fr = LightevalTaskConfig( - name="wmt12:en-fr", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt12_en-fr", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt12_es_en = LightevalTaskConfig( - name="wmt12:es-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt12_es-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt12_fr_en = LightevalTaskConfig( - name="wmt12:fr-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt12_fr-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt13_cs_en = LightevalTaskConfig( - name="wmt13:cs-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt13_cs-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt13_de_en = LightevalTaskConfig( - name="wmt13:de-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt13_de-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt13_en_cs = LightevalTaskConfig( - name="wmt13:en-cs", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt13_en-cs", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt13_en_de = LightevalTaskConfig( - name="wmt13:en-de", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt13_en-de", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt13_en_es = LightevalTaskConfig( - name="wmt13:en-es", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt13_en-es", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt13_en_fr = LightevalTaskConfig( - name="wmt13:en-fr", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt13_en-fr", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt13_en_ru = LightevalTaskConfig( - name="wmt13:en-ru", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt13_en-ru", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt13_es_en = LightevalTaskConfig( - name="wmt13:es-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt13_es-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt13_fr_en = LightevalTaskConfig( - name="wmt13:fr-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt13_fr-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt13_ru_en = LightevalTaskConfig( - name="wmt13:ru-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt13_ru-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt14_cs_en = LightevalTaskConfig( - name="wmt14:cs-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt14_cs-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt14_de_en = LightevalTaskConfig( - name="wmt14:de-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt14_de-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt14_en_cs = LightevalTaskConfig( - name="wmt14:en-cs", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt14_en-cs", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt14_en_de = LightevalTaskConfig( - name="wmt14:en-de", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt14_en-de", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt14_en_fr = LightevalTaskConfig( - name="wmt14:en-fr", - prompt_function=wmt_alphabetical_prompt, - hf_repo="wmt14", - hf_subset="fr-en", - hf_avail_splits=["train", "validation", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt14_en_fr = LightevalTaskConfig( - name="wmt14:en-fr", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt14_en-fr", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt14_en_hi = LightevalTaskConfig( - name="wmt14:en-hi", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt14_en-hi", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt14_en_ru = LightevalTaskConfig( - name="wmt14:en-ru", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt14_en-ru", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt14_fr_en = LightevalTaskConfig( - name="wmt14:fr-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="wmt14", - hf_subset="fr-en", - hf_avail_splits=["train", "validation", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt14_fr_en = LightevalTaskConfig( - name="wmt14:fr-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt14_fr-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt14_hi_en = LightevalTaskConfig( - name="wmt14:hi-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt14_hi-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt14_ru_en = LightevalTaskConfig( - name="wmt14:ru-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt14_ru-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt15_cs_en = LightevalTaskConfig( - name="wmt15:cs-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt15_cs-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt15_de_en = LightevalTaskConfig( - name="wmt15:de-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt15_de-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt15_en_cs = LightevalTaskConfig( - name="wmt15:en-cs", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt15_en-cs", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt15_en_de = LightevalTaskConfig( - name="wmt15:en-de", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt15_en-de", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt15_en_fi = LightevalTaskConfig( - name="wmt15:en-fi", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt15_en-fi", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt15_en_fr = LightevalTaskConfig( - name="wmt15:en-fr", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt15_en-fr", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt15_en_ru = LightevalTaskConfig( - name="wmt15:en-ru", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt15_en-ru", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt15_fi_en = LightevalTaskConfig( - name="wmt15:fi-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt15_fi-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt15_fr_en = LightevalTaskConfig( - name="wmt15:fr-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt15_fr-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt15_ru_en = LightevalTaskConfig( - name="wmt15:ru-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt15_ru-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt16_cs_en = LightevalTaskConfig( - name="wmt16:cs-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt16_cs-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt16_de_en = LightevalTaskConfig( - name="wmt16:de-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="wmt16", - hf_subset="de-en", - hf_avail_splits=["train", "validation", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt16_de_en = LightevalTaskConfig( - name="wmt16:de-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt16_de-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt16_en_cs = LightevalTaskConfig( - name="wmt16:en-cs", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt16_en-cs", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt16_en_de = LightevalTaskConfig( - name="wmt16:en-de", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="wmt16", - hf_subset="de-en", - hf_avail_splits=["train", "validation", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt16_en_de = LightevalTaskConfig( - name="wmt16:en-de", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt16_en-de", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt16_en_fi = LightevalTaskConfig( - name="wmt16:en-fi", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt16_en-fi", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt16_en_ro = LightevalTaskConfig( - name="wmt16:en-ro", - prompt_function=wmt_alphabetical_prompt, - hf_repo="wmt16", - hf_subset="ro-en", - hf_avail_splits=["train", "validation", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt16_en_ro = LightevalTaskConfig( - name="wmt16:en-ro", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt16_en-ro", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt16_en_ru = LightevalTaskConfig( - name="wmt16:en-ru", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt16_en-ru", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt16_en_tr = LightevalTaskConfig( - name="wmt16:en-tr", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt16_en-tr", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt16_fi_en = LightevalTaskConfig( - name="wmt16:fi-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt16_fi-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt16_ro_en = LightevalTaskConfig( - name="wmt16:ro-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="wmt16", - hf_subset="ro-en", - hf_avail_splits=["train", "validation", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt16_ro_en = LightevalTaskConfig( - name="wmt16:ro-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt16_ro-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt16_ru_en = LightevalTaskConfig( - name="wmt16:ru-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt16_ru-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt16_tr_en = LightevalTaskConfig( - name="wmt16:tr-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt16_tr-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt17_cs_en = LightevalTaskConfig( - name="wmt17:cs-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt17_cs-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt17_de_en = LightevalTaskConfig( - name="wmt17:de-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt17_de-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt17_en_cs = LightevalTaskConfig( - name="wmt17:en-cs", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt17_en-cs", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt17_en_de = LightevalTaskConfig( - name="wmt17:en-de", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt17_en-de", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt17_en_fi = LightevalTaskConfig( - name="wmt17:en-fi", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt17_en-fi", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt17_en_lv = LightevalTaskConfig( - name="wmt17:en-lv", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt17_en-lv", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt17_en_ru = LightevalTaskConfig( - name="wmt17:en-ru", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt17_en-ru", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt17_en_tr = LightevalTaskConfig( - name="wmt17:en-tr", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt17_en-tr", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt17_en_zh = LightevalTaskConfig( - name="wmt17:en-zh", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt17_en-zh", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt17_fi_en = LightevalTaskConfig( - name="wmt17:fi-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt17_fi-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt17_lv_en = LightevalTaskConfig( - name="wmt17:lv-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt17_lv-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt17_ru_en = LightevalTaskConfig( - name="wmt17:ru-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt17_ru-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt17_tr_en = LightevalTaskConfig( - name="wmt17:tr-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt17_tr-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt17_zh_en = LightevalTaskConfig( - name="wmt17:zh-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt17_zh-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt18_cs_en = LightevalTaskConfig( - name="wmt18:cs-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt18_cs-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt18_de_en = LightevalTaskConfig( - name="wmt18:de-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt18_de-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt18_en_cs = LightevalTaskConfig( - name="wmt18:en-cs", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt18_en-cs", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt18_en_de = LightevalTaskConfig( - name="wmt18:en-de", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt18_en-de", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt18_en_et = LightevalTaskConfig( - name="wmt18:en-et", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt18_en-et", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt18_en_fi = LightevalTaskConfig( - name="wmt18:en-fi", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt18_en-fi", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt18_en_ru = LightevalTaskConfig( - name="wmt18:en-ru", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt18_en-ru", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt18_en_tr = LightevalTaskConfig( - name="wmt18:en-tr", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt18_en-tr", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt18_en_zh = LightevalTaskConfig( - name="wmt18:en-zh", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt18_en-zh", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt18_et_en = LightevalTaskConfig( - name="wmt18:et-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt18_et-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt18_fi_en = LightevalTaskConfig( - name="wmt18:fi-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt18_fi-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt18_ru_en = LightevalTaskConfig( - name="wmt18:ru-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt18_ru-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt18_tr_en = LightevalTaskConfig( - name="wmt18:tr-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt18_tr-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt18_zh_en = LightevalTaskConfig( - name="wmt18:zh-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt18_zh-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt19_cs_de = LightevalTaskConfig( - name="wmt19:cs-de", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt19_cs-de", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt19_de_cs = LightevalTaskConfig( - name="wmt19:de-cs", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt19_de-cs", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt19_de_en = LightevalTaskConfig( - name="wmt19:de-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt19_de-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt19_de_fr = LightevalTaskConfig( - name="wmt19:de-fr", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt19_de-fr", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt19_en_cs = LightevalTaskConfig( - name="wmt19:en-cs", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt19_en-cs", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt19_en_de = LightevalTaskConfig( - name="wmt19:en-de", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt19_en-de", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt19_en_fi = LightevalTaskConfig( - name="wmt19:en-fi", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt19_en-fi", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt19_en_gu = LightevalTaskConfig( - name="wmt19:en-gu", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt19_en-gu", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt19_en_kk = LightevalTaskConfig( - name="wmt19:en-kk", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt19_en-kk", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt19_en_lt = LightevalTaskConfig( - name="wmt19:en-lt", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt19_en-lt", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt19_en_ru = LightevalTaskConfig( - name="wmt19:en-ru", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt19_en-ru", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt19_en_zh = LightevalTaskConfig( - name="wmt19:en-zh", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt19_en-zh", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt19_fi_en = LightevalTaskConfig( - name="wmt19:fi-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt19_fi-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt19_fr_de = LightevalTaskConfig( - name="wmt19:fr-de", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt19_fr-de", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt19_gu_en = LightevalTaskConfig( - name="wmt19:gu-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt19_gu-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt19_kk_en = LightevalTaskConfig( - name="wmt19:kk-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt19_kk-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt19_lt_en = LightevalTaskConfig( - name="wmt19:lt-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt19_lt-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt19_ru_en = LightevalTaskConfig( - name="wmt19:ru-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt19_ru-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt19_zh_en = LightevalTaskConfig( - name="wmt19:zh-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt19_zh-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_cs_en = LightevalTaskConfig( - name="wmt20:cs-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_cs-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_de_en = LightevalTaskConfig( - name="wmt20:de-en", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_de-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_de_fr = LightevalTaskConfig( - name="wmt20:de-fr", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_de-fr", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_en_cs = LightevalTaskConfig( - name="wmt20:en-cs", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_en-cs", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_en_de = LightevalTaskConfig( - name="wmt20:en-de", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_en-de", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_en_iu = LightevalTaskConfig( - name="wmt20:en-iu", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_en-iu", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_en_ja = LightevalTaskConfig( - name="wmt20:en-ja", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_en-ja", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_en_km = LightevalTaskConfig( - name="wmt20:en-km", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_en-km", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_en_pl = LightevalTaskConfig( - name="wmt20:en-pl", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_en-pl", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_en_ps = LightevalTaskConfig( - name="wmt20:en-ps", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_en-ps", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_en_ru = LightevalTaskConfig( - name="wmt20:en-ru", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_en-ru", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_en_ta = LightevalTaskConfig( - name="wmt20:en-ta", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_en-ta", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_en_zh = LightevalTaskConfig( - name="wmt20:en-zh", - prompt_function=wmt_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_en-zh", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_fr_de = LightevalTaskConfig( - name="wmt20:fr-de", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_fr-de", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_iu_en = LightevalTaskConfig( - name="wmt20:iu-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_iu-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_ja_en = LightevalTaskConfig( - name="wmt20:ja-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_ja-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_km_en = LightevalTaskConfig( - name="wmt20:km-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_km-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_pl_en = LightevalTaskConfig( - name="wmt20:pl-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_pl-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_ps_en = LightevalTaskConfig( - name="wmt20:ps-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_ps-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_ru_en = LightevalTaskConfig( - name="wmt20:ru-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_ru-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_ta_en = LightevalTaskConfig( - name="wmt20:ta-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_ta-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -wmt20_zh_en = LightevalTaskConfig( - name="wmt20:zh-en", - prompt_function=wmt_reverse_alphabetical_prompt, - hf_repo="lighteval/sacrebleu_manual", - hf_subset="wmt20_zh-en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=None, - metrics=[Metrics.bleu, Metrics.chrf, Metrics.ter], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - wmt14_de_en, - wmt16_en_cs, - wmt19_en_cs, - wmt19_en_de, - wmt19_en_fi, - wmt19_en_gu, - wmt19_en_kk, - wmt19_en_lt, - wmt19_en_ru, - wmt19_en_zh, - wmt19_fi_en, - wmt19_fr_de, - wmt19_gu_en, - wmt19_kk_en, - wmt19_lt_en, - wmt19_ru_en, - wmt19_zh_en, - wmt20_cs_en, - wmt20_de_en, - wmt20_en_de, - wmt20_en_iu, - wmt20_en_ja, - wmt20_en_km, - wmt20_en_pl, - wmt20_en_ps, - wmt20_en_ru, - wmt20_en_ta, - wmt20_en_zh, - wmt20_fr_de, - wmt20_iu_en, - wmt20_ja_en, - wmt20_km_en, - wmt20_pl_en, - wmt20_ps_en, - wmt20_ru_en, - wmt20_ta_en, - wmt20_zh_en, -] diff --git a/src/lighteval/tasks/tasks/sciq.py b/src/lighteval/tasks/tasks/sciq.py deleted file mode 100644 index 5e6ed56b9..000000000 --- a/src/lighteval/tasks/tasks/sciq.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -name: -Sciq - -dataset: -allenai/sciq - -abstract: -The SciQ dataset contains 13,679 crowdsourced science exam questions about -Physics, Chemistry and Biology, among others. The questions are in -multiple-choice format with 4 answer options each. For the majority of the -questions, an additional paragraph with supporting evidence for the correct -answer is provided. - -languages: -english - -tags: -physics, chemistry, biology, reasoning, multiple-choice, qa - -paper: -https://arxiv.org/abs/1707.06209 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def sciq_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"{line['support']}\nQuestion: {line['question']}\nAnswer:".strip(), - choices=[ - f" {c}" for c in [line["distractor1"], line["distractor2"], line["distractor3"], line["correct_answer"]] - ], - gold_index=3, - ) - - -sciq = LightevalTaskConfig( - name="sciq", - prompt_function=sciq_prompt, - hf_repo="allenai/sciq", - hf_subset="default", - hf_avail_splits=["train", "validation", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - sciq, -] diff --git a/src/lighteval/tasks/tasks/simpleqa.py b/src/lighteval/tasks/tasks/simpleqa.py deleted file mode 100644 index 4493fc322..000000000 --- a/src/lighteval/tasks/tasks/simpleqa.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -name: -Simpleqa - -dataset: -lighteval/SimpleQA - -abstract: -A factuality benchmark called SimpleQA that measures the ability for language -models to answer short, fact-seeking questions. - -languages: -english - -tags: -factuality, general-knowledge, qa - -paper: -https://openai.com/index/introducing-simpleqa/ - -starred: -true -""" - -from inspect_ai.dataset import Sample -from inspect_ai.scorer import model_graded_fact -from inspect_ai.solver import generate - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def simpleqa_prompt(line, task_name: str = None): - query = f"Question: {line['question']}\n" - query += "".join( - [f"\n{key}. {choice}" for key, choice in zip(["A", "B", "C", "D", "E", "F"], line["choices"]["text"])] - ) - query += "\nAnswer:" - return Doc( - task_name=task_name, - query=query, - choices=line["choices"]["text"], - gold_index=line["choices"]["label"].index(line["answerKey"]), - ) - - -def record_to_sample(record): - query = record["problem"] - target = record["answer"] - return Sample(input=query, target=target) - - -simpleqa = LightevalTaskConfig( - name="simpleqa", - prompt_function=simpleqa_prompt, - hf_repo="lighteval/SimpleQA", - hf_subset="default", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split="few_shot", - few_shots_select=None, - generation_size=2048, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, - sample_fields=record_to_sample, - solver=[generate(cache=True)], - scorer=model_graded_fact(), -) - -TASKS_TABLE = [ - simpleqa, -] diff --git a/src/lighteval/tasks/tasks/siqa.py b/src/lighteval/tasks/tasks/siqa.py deleted file mode 100644 index d0d22b8ec..000000000 --- a/src/lighteval/tasks/tasks/siqa.py +++ /dev/null @@ -1,75 +0,0 @@ -""" -name: -Siqa - -dataset: -allenai/social_i_qa - -abstract: -We introduce Social IQa: Social Interaction QA, a new question-answering -benchmark for testing social commonsense intelligence. Contrary to many prior -benchmarks that focus on physical or taxonomic knowledge, Social IQa focuses on -reasoning about people's actions and their social implications. For example, -given an action like "Jesse saw a concert" and a question like "Why did Jesse do -this?", humans can easily infer that Jesse wanted "to see their favorite -performer" or "to enjoy the music", and not "to see what's happening inside" or -"to see if it works". The actions in Social IQa span a wide variety of social -situations, and answer candidates contain both human-curated answers and -adversarially-filtered machine-generated candidates. Social IQa contains over -37,000 QA pairs for evaluating models' abilities to reason about the social -implications of everyday events and situations. - -languages: -english - -tags: -commonsense, multiple-choice, qa - -paper: -""" - -from string import ascii_uppercase - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def siqa_prompt(line, task_name: str = None): - query = "The following are multiple choice questions (with answers) about common sense.\n" - query += f"Question: {line['context']} {line['question']}\n" - query += "".join( - [ - f"{key}. {choice}\n" - for key, choice in zip(list(ascii_uppercase)[:3], [line["answerA"], line["answerB"], line["answerC"]]) - ] - ) - query += "Answer: " - - return Doc( - task_name=task_name, - query=query, - choices=["A", "B", "C"], - gold_index=int(line["label"]) - 1, - instruction="The following are multiple choice questions (with answers) about common sense.\n", - ) - - -siqa = LightevalTaskConfig( - name="siqa", - prompt_function=siqa_prompt, - hf_repo="allenai/social_i_qa", - hf_subset="default", - hf_avail_splits=["train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=1, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - siqa, -] diff --git a/src/lighteval/tasks/tasks/slr_bench.py b/src/lighteval/tasks/tasks/slr_bench.py deleted file mode 100644 index c17fdffe7..000000000 --- a/src/lighteval/tasks/tasks/slr_bench.py +++ /dev/null @@ -1,119 +0,0 @@ -""" -name: -SLR-Bench - -dataset: -AIML-TUDA/SLR-Bench - -abstract: -SLR-Bench is a large-scale benchmark for scalable logical reasoning with -language models, comprising 19,000 prompts organized into 20 curriculum levels. - -languages: -english - -tags: -reasoning, symbolic - -paper: -https://arxiv.org/abs/2506.15787 -""" - -import logging - -import numpy as np - -from lighteval.metrics.utils.metric_utils import SampleLevelComputation, SampleLevelMetric -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc, SamplingMethod -from lighteval.utils.imports import is_package_available, requires - - -if is_package_available("evaluate"): - from evaluate import load -else: - load = None - -logger = logging.getLogger(__name__) - - -@requires("evaluate") -def prompt_fn(line: dict, task_name: str): - """Defines how to go from a dataset line to a doc object.""" - # Check for SWI-Prolog installation - import shutil - - if shutil.which("swipl") is None: - raise ImportError( - "SWI-Prolog (swipl) is not installed or not in PATH. Please install SWI-Prolog to use this task. " - ) - - return Doc( - task_name=task_name, query=line["prompt"], choices=[str(line.get("validation program", ""))], gold_index=0 - ) - - -class VerifiableRewardMetric(SampleLevelComputation): - # Load the symbolic judge for evaluating Prolog programs - - def compute(self, doc, model_response, **kwargs): - symbolic_judge = load("AIML-TUDA/VerifiableRewardsForScalableLogicalReasoning") - try: - prediction = model_response.final_text[0] - validation_program = doc.choices[0] if doc.choices else "" - ref_format = [ - { - "validation_program": validation_program, - "evaluation_config": {"positive_predicate": "eastbound", "negative_predicate": "westbound"}, - } - ] - - results = symbolic_judge.compute(predictions=[prediction], references=ref_format) - return results["accuracy"] - - except Exception as e: - logger.error("Error during the computation of the metric") - raise RuntimeError(f"Failed to compute verifiable reward metric: {e}") - - -custom_metric = SampleLevelMetric( - metric_name="verifiable_reward", - higher_is_better=True, - category=SamplingMethod.GENERATIVE, - sample_level_fn=VerifiableRewardMetric(), - corpus_level_fn=np.mean, -) - -# Define the subsets available in the SLR-Bench dataset -CONFIGURATIONS = ["All", "Basic", "Easy", "Medium", "Hard"] - - -class SLRBenchTask(LightevalTaskConfig): - """Task configuration for SLR-Bench evaluation.""" - - def __init__( - self, - config: str, - ): - name = f"slr_bench_{config.lower()}" - super().__init__( - name=name, - hf_subset=f"v1-{config}", - prompt_function=prompt_fn, - hf_repo="AIML-TUDA/SLR-Bench", - metrics=[custom_metric], - hf_avail_splits=["train", "validation", "test"], - evaluation_splits=["test"], - few_shots_split="train", - few_shots_select="random_sampling_from_train", - generation_size=4096, - stop_sequence=None, - version=1, - ) - - -# Create a single task instance for each configuration -TASKS = [SLRBenchTask(config) for config in CONFIGURATIONS] - -# Export tasks table -TASKS_TABLE = TASKS diff --git a/src/lighteval/tasks/tasks/squad_v2.py b/src/lighteval/tasks/tasks/squad_v2.py deleted file mode 100644 index a70358b65..000000000 --- a/src/lighteval/tasks/tasks/squad_v2.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -name: -Squad V2 - -dataset: -rajpurkar/squad_v2 - -abstract: -Stanford Question Answering Dataset (SQuAD) is a reading comprehension dataset, -consisting of questions posed by crowdworkers on a set of Wikipedia articles, -where the answer to every question is a segment of text, or span, from the -corresponding reading passage, or the question might be unanswerable. -SQuAD 2.0 combines the 100,000 questions in SQuAD1.1 with over 50,000 -unanswerable questions written adversarially by crowdworkers to look similar to -answerable ones. To do well on SQuAD2.0, systems must not only answer questions -when possible, but also determine when no answer is supported by the paragraph -and abstain from answering. - -languages: -english - -tags: -qa - -paper: -https://arxiv.org/abs/1806.03822 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.templates.qa import get_qa_prompt_function -from lighteval.utils.language import Language - - -squad_v2 = LightevalTaskConfig( - name="squad_v2", - prompt_function=get_qa_prompt_function( - Language.ENGLISH, - lambda line: { - "question": line["question"], - "context": line["context"], - "choices": [ans for ans in line["answers"]["text"] if len(ans) > 0], - }, - ), - hf_repo="rajpurkar/squad_v2", - hf_subset="squad_v2", - hf_filter=lambda line: any(ans for ans in line["answers"]["text"] if len(ans) > 0), - evaluation_splits=("validation",), - few_shots_split="train", - stop_sequence=["\n", "Question:", "question:"], - generation_size=200, - metrics=[Metrics.exact_match], - version=1, -) - -TASKS_TABLE = [ - squad_v2, -] diff --git a/src/lighteval/tasks/tasks/storycloze.py b/src/lighteval/tasks/tasks/storycloze.py deleted file mode 100644 index 233f8f0cf..000000000 --- a/src/lighteval/tasks/tasks/storycloze.py +++ /dev/null @@ -1,70 +0,0 @@ -""" -name: -Storycloze - -dataset: -MoE-UNC/story_cloze - -abstract: -A Corpus and Cloze Evaluation for Deeper Understanding of -Commonsense Stories - -languages: -english - -tags: -narrative, reasoning - -paper: -https://arxiv.org/abs/1604.01696 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def storycloze_prompt(line, task_name: str = None): - context = "\n".join( - [line["input_sentence_1"], line["input_sentence_2"], line["input_sentence_3"], line["input_sentence_4"]] - ) - choices = [line["sentence_quiz1"], line["sentence_quiz2"]] - gold = int(line["answer_right_ending"]) - 1 - return Doc(task_name=task_name, query=context + "\n", choices=choices, gold_index=gold) - - -storycloze_2016 = LightevalTaskConfig( - name="storycloze:2016", - prompt_function=storycloze_prompt, - hf_repo="MoE-UNC/story_cloze", - hf_subset="2016", - hf_avail_splits=["validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - - -storycloze_2018 = LightevalTaskConfig( - name="storycloze:2018", - prompt_function=storycloze_prompt, - hf_repo="MoE-UNC/story_cloze", - hf_subset="2018", - hf_avail_splits=["validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - storycloze_2016, - storycloze_2018, -] diff --git a/src/lighteval/tasks/tasks/summarization.py b/src/lighteval/tasks/tasks/summarization.py deleted file mode 100644 index 4bc394395..000000000 --- a/src/lighteval/tasks/tasks/summarization.py +++ /dev/null @@ -1,119 +0,0 @@ -""" -name: -Summarization - -dataset: -lighteval/summarization - -abstract: -Don't Give Me the Details, Just the Summary! Topic-Aware Convolutional Neural -Networks for Extreme Summarization and: Abstractive Text Summarization using -Sequence-to-sequence RNNs and Beyond - -languages: -english - -tags: -summarization - -paper: -https://aclanthology.org/D18-1206/ -https://aclanthology.org/K16-1028/ -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def cnn_dm_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"Article: {line['article']}\n\nTL;DR:", - choices=[line["highlights"]], - gold_index=0, - ) - - -def xsum_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"Document: {line['document']}\n\nA one-sentence summary of the above document is:", - choices=[line["summary"]], - gold_index=0, - ) - - -summarization_cnn_dm = LightevalTaskConfig( - name="summarization:cnn-dm", - prompt_function=cnn_dm_prompt, - hf_repo="lighteval/summarization", - hf_subset="cnn-dm", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=128, - metrics=[ - Metrics.rouge1, - Metrics.rouge2, - Metrics.rougeL, - Metrics.faithfulness, - Metrics.extractiveness, - Metrics.bert_score, - ], - stop_sequence=["\n"], - version=0, -) - - -summarization_xsum = LightevalTaskConfig( - name="summarization:xsum", - prompt_function=xsum_prompt, - hf_repo="lighteval/summarization", - hf_subset="xsum", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=64, - metrics=[ - Metrics.rouge1, - Metrics.rouge2, - Metrics.rougeL, - Metrics.faithfulness, - Metrics.extractiveness, - Metrics.bert_score, - ], - stop_sequence=["\n"], - version=0, -) - - -summarization_xsum_sampled = LightevalTaskConfig( - name="summarization:xsum-sampled", - prompt_function=xsum_prompt, - hf_repo="lighteval/summarization", - hf_subset="xsum-sampled", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=64, - metrics=[ - Metrics.rouge1, - Metrics.rouge2, - Metrics.rougeL, - Metrics.faithfulness, - Metrics.extractiveness, - Metrics.bert_score, - ], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - summarization_cnn_dm, - summarization_xsum, - summarization_xsum_sampled, -] diff --git a/src/lighteval/tasks/tasks/swag.py b/src/lighteval/tasks/tasks/swag.py deleted file mode 100644 index 67febe8ec..000000000 --- a/src/lighteval/tasks/tasks/swag.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -name: -Swag - -dataset: -allenai/swag - -abstract: -The dataset consists of 113k multiple choice questions about grounded situations -(73k training, 20k validation, 20k test). Each question is a video caption from -LSMDC or ActivityNet Captions, with four answer choices about what might happen -next in the scene. The correct answer is the (real) video caption for the next -event in the video; the three incorrect answers are adversarially generated and -human verified, so as to fool machines but not humans. SWAG aims to be a -benchmark for evaluating grounded commonsense NLI and for learning -representations. - -languages: -english - -tags: -narrative, reasoning - -paper: -https://arxiv.org/abs/1808.05326 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def swag_prompt(line, task_name: str = None): - choices = [line["ending0"], line["ending1"], line["ending2"], line["ending3"]] - return Doc( - task_name=task_name, - query=line["startphrase"], - choices=choices, - gold_index=int(line["label"]), - ) - - -swag = LightevalTaskConfig( - name="swag", - prompt_function=swag_prompt, - hf_repo="allenai/swag", - hf_subset="regular", - hf_avail_splits=["train", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - swag, -] diff --git a/src/lighteval/tasks/tasks/synthetic_reasoning.py b/src/lighteval/tasks/tasks/synthetic_reasoning.py deleted file mode 100644 index c4ac36a99..000000000 --- a/src/lighteval/tasks/tasks/synthetic_reasoning.py +++ /dev/null @@ -1,137 +0,0 @@ -""" -name: -Synthetic Reasoning - -dataset: -lighteval/synthetic_reasoning, lighteval/synthetic_reasoning_natural - -abstract: -LIME: Learning Inductive Bias for Primitives of Mathematical Reasoning - -languages: -english - -tags: -reasoning - -paper: -https://arxiv.org/abs/2206.03855 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def synthetic_reasoning_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"Please solve the following problem.\n\n{line['source']}\nTarget: ", - gold_index=0, - choices=[line["target"]], - instruction="Please solve the following problem.\n\n", - ) - - -def synthetic_reasoning_natural_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"Please solve the following problem.\n\nRules: \n{line['question']}", - gold_index=0, - choices=[line["target"]], - instruction="Please solve the following problem.\n\n", - ) - - -synthetic_reasoning_induction = LightevalTaskConfig( - name="synthetic_reasoning:induction", - prompt_function=synthetic_reasoning_prompt, - hf_repo="lighteval/synthetic_reasoning", - hf_subset="induction", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=50, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - - -synthetic_reasoning_natural_easy = LightevalTaskConfig( - name="synthetic_reasoning:natural_easy", - prompt_function=synthetic_reasoning_natural_prompt, - hf_repo="lighteval/synthetic_reasoning_natural", - hf_subset="easy", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - - -synthetic_reasoning_natural_hard = LightevalTaskConfig( - name="synthetic_reasoning:natural_hard", - prompt_function=synthetic_reasoning_natural_prompt, - hf_repo="lighteval/synthetic_reasoning_natural", - hf_subset="hard", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - - -synthetic_reasoning_pattern_match = LightevalTaskConfig( - name="synthetic_reasoning:pattern_match", - prompt_function=synthetic_reasoning_prompt, - hf_repo="lighteval/synthetic_reasoning", - hf_subset="pattern_match", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=50, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - - -synthetic_reasoning_variable_substitution = LightevalTaskConfig( - name="synthetic_reasoning:variable_substitution", - prompt_function=synthetic_reasoning_prompt, - hf_repo="lighteval/synthetic_reasoning", - hf_subset="variable_substitution", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation", "test"], - few_shots_split=None, - few_shots_select=None, - generation_size=50, - metrics=[ - Metrics.exact_match, - ], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - synthetic_reasoning_induction, - synthetic_reasoning_natural_easy, - synthetic_reasoning_natural_hard, - synthetic_reasoning_pattern_match, - synthetic_reasoning_variable_substitution, -] diff --git a/src/lighteval/tasks/tasks/the_pile.py b/src/lighteval/tasks/tasks/the_pile.py deleted file mode 100644 index abd7f5f9a..000000000 --- a/src/lighteval/tasks/tasks/the_pile.py +++ /dev/null @@ -1,336 +0,0 @@ -""" -name: -The Pile - -dataset: -lighteval/pile_helm - -abstract: -The Pile corpus for measuring lanugage model performance across various domains. - -languages: -english - -tags: -language-modeling - -paper: -https://arxiv.org/abs/2101.00027 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def the_pile_prompt(line, task_name: str = None): - return Doc(task_name=task_name, query=line["text"], gold_index=None, choices=None) - - -the_pile_arxiv_helm = LightevalTaskConfig( - name="the_pile:arxiv", - prompt_function=the_pile_prompt, - hf_repo="lighteval/pile_helm", - hf_subset="arxiv", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - -the_pile_bibliotik_helm = LightevalTaskConfig( - name="the_pile:bibliotik", - prompt_function=the_pile_prompt, - hf_repo="lighteval/pile_helm", - hf_subset="bibliotik", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - -the_pile_commoncrawl_helm = LightevalTaskConfig( - name="the_pile:commoncrawl", - prompt_function=the_pile_prompt, - hf_repo="lighteval/pile_helm", - hf_subset="commoncrawl", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - -the_pile_dm_mathematics_helm = LightevalTaskConfig( - name="the_pile:dm-mathematics", - prompt_function=the_pile_prompt, - hf_repo="lighteval/pile_helm", - hf_subset="dm-mathematics", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - -the_pile_enron_helm = LightevalTaskConfig( - name="the_pile:enron", - prompt_function=the_pile_prompt, - hf_repo="lighteval/pile_helm", - hf_subset="enron", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - -the_pile_europarl_helm = LightevalTaskConfig( - name="the_pile:europarl", - prompt_function=the_pile_prompt, - hf_repo="lighteval/pile_helm", - hf_subset="europarl", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - -the_pile_freelaw_helm = LightevalTaskConfig( - name="the_pile:freelaw", - prompt_function=the_pile_prompt, - hf_repo="lighteval/pile_helm", - hf_subset="freelaw", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - -the_pile_github_helm = LightevalTaskConfig( - name="the_pile:github", - prompt_function=the_pile_prompt, - hf_repo="lighteval/pile_helm", - hf_subset="github", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - -the_pile_gutenberg_helm = LightevalTaskConfig( - name="the_pile:gutenberg", - prompt_function=the_pile_prompt, - hf_repo="lighteval/pile_helm", - hf_subset="gutenberg", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - -the_pile_hackernews_helm = LightevalTaskConfig( - name="the_pile:hackernews", - prompt_function=the_pile_prompt, - hf_repo="lighteval/pile_helm", - hf_subset="hackernews", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - -the_pile_nih_exporter_helm = LightevalTaskConfig( - name="the_pile:nih-exporter", - prompt_function=the_pile_prompt, - hf_repo="lighteval/pile_helm", - hf_subset="nih-exporter", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - -the_pile_opensubtitles_helm = LightevalTaskConfig( - name="the_pile:opensubtitles", - prompt_function=the_pile_prompt, - hf_repo="lighteval/pile_helm", - hf_subset="opensubtitles", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - -the_pile_openwebtext2_helm = LightevalTaskConfig( - name="the_pile:openwebtext2", - prompt_function=the_pile_prompt, - hf_repo="lighteval/pile_helm", - hf_subset="openwebtext2", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - - -the_pile_pubmed_abstracts_helm = LightevalTaskConfig( - name="the_pile:pubmed-abstracts", - prompt_function=the_pile_prompt, - hf_repo="lighteval/pile_helm", - hf_subset="pubmed-abstracts", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - -the_pile_pubmed_central_helm = LightevalTaskConfig( - name="the_pile:pubmed-central", - prompt_function=the_pile_prompt, - hf_repo="lighteval/pile_helm", - hf_subset="pubmed-central", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - -the_pile_stackexchange_helm = LightevalTaskConfig( - name="the_pile:stackexchange", - prompt_function=the_pile_prompt, - hf_repo="lighteval/pile_helm", - hf_subset="stackexchange", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - -the_pile_upsto_helm = LightevalTaskConfig( - name="the_pile:upsto", - prompt_function=the_pile_prompt, - hf_repo="lighteval/pile_helm", - hf_subset="uspto", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - -the_pile_wikipedia_helm = LightevalTaskConfig( - name="the_pile:wikipedia", - prompt_function=the_pile_prompt, - hf_repo="lighteval/pile_helm", - hf_subset="wikipedia", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - -the_pile_youtubesubtitles_helm = LightevalTaskConfig( - name="the_pile:youtubesubtitles", - prompt_function=the_pile_prompt, - hf_repo="lighteval/pile_helm", - hf_subset="youtubesubtitles", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - the_pile_arxiv_helm, - the_pile_bibliotik_helm, - the_pile_commoncrawl_helm, - the_pile_dm_mathematics_helm, - the_pile_enron_helm, - the_pile_europarl_helm, - the_pile_freelaw_helm, - the_pile_github_helm, - the_pile_gutenberg_helm, - the_pile_hackernews_helm, - the_pile_nih_exporter_helm, - the_pile_opensubtitles_helm, - the_pile_openwebtext2_helm, - the_pile_pubmed_abstracts_helm, - the_pile_pubmed_central_helm, - the_pile_stackexchange_helm, - the_pile_upsto_helm, - the_pile_wikipedia_helm, - the_pile_youtubesubtitles_helm, -] diff --git a/src/lighteval/tasks/tasks/tiny_benchmarks.py b/src/lighteval/tasks/tasks/tiny_benchmarks.py deleted file mode 100644 index afaff8f08..000000000 --- a/src/lighteval/tasks/tasks/tiny_benchmarks.py +++ /dev/null @@ -1,282 +0,0 @@ -""" -name: -Tiny Benchmarks - -dataset: -tinyBenchmarks/tinyWinogrande, tinyBenchmarks/tinyAI2_arc, -tinyBenchmarks/tinyHellaswag, tinyBenchmarks/tinyMMLU, -tinyBenchmarks/tinyTruthfulQA, tinyBenchmarks/tinyGSM8k - -abstract: -TinyBenchmarks is a benchmark for evaluating the performance of language models -on tiny benchmarks. - -languages: -english - -tags: -general-knowledge, reasoning, qa - -paper: -https://arxiv.org/abs/2402.14992 -""" - -import os -import pathlib -import pickle - -import numpy as np -import requests -from scipy.optimize import minimize - -from lighteval.metrics.metrics import CorpusLevelMetricGrouping -from lighteval.metrics.metrics_corpus import CorpusLevelComputation -from lighteval.metrics.metrics_sample import ExactMatches, LoglikelihoodAcc, SampleLevelComputation -from lighteval.metrics.normalizations import gsm8k_normalizer -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc, SamplingMethod -from lighteval.tasks.tasks.arc import arc_prompt -from lighteval.tasks.tasks.gsm8k import gsm8k_prompt -from lighteval.tasks.tasks.hellaswag import hellaswag_prompt -from lighteval.tasks.tasks.mmlu import mmlu_prompt -from lighteval.tasks.tasks.truthfulqa import truthful_qa_multiple_choice_prompt -from lighteval.tasks.tasks.winogrande import winogrande_prompt - - -# Utility functions -def sigmoid(z): - return 1 / (1 + np.exp(-z)) - - -def item_curve(theta, a, b): - z = np.clip(a * theta - b, -30, 30).sum(axis=1) - return sigmoid(z) - - -def fit_theta(responses_test, seen_items, A, B, theta_init=None, eps=1e-10, optimizer="BFGS"): - D = A.shape[1] - - # Define the negative log likelihood function - def neg_log_like(x): - P = item_curve(x.reshape(1, D, 1), A[:, :, seen_items], B[:, :, seen_items]).squeeze() - log_likelihood = np.sum( - responses_test[seen_items] * np.log(P + eps) + (1 - responses_test[seen_items]) * np.log(1 - P + eps) - ) - return -log_likelihood - - # Use the minimize function to find the ability parameters that minimize the negative log likelihood - optimal_theta = minimize(neg_log_like, np.zeros(D), method=optimizer).x[None, :, None] - return optimal_theta - - -# Evaluation function -class TinyCorpusAggregator(SampleLevelComputation, CorpusLevelComputation): - LEADEBRBOARD_SCENARIOS = ["truthfulqa", "gsm8k", "winogrande", "arc", "hellaswag"] - BENCHS = ["lb", "mmlu"] - METRICS = ["irt", "pirt", "gpirt"] - # Not included yet: - # - helm_lite (not avail on datasets) - # - alpaca (needs to be added to lighteval first) - - def __init__(self, task: str): - self.number_of_examples = 100 - if task not in self.LEADEBRBOARD_SCENARIOS + self.BENCHS: - raise ValueError(f"Bench name must be one of {','.join(self.LEADEBRBOARD_SCENARIOS + self.BENCHS)}.") - self.task = task - self.scenario = "lb" if task in self.LEADEBRBOARD_SCENARIOS else task - self.download() - self.estimates = None - self.num_samples = 0 - - def download(self): - # Likely to crash in // processes if we don't include the pkl - path_dld = os.path.join(pathlib.Path(__file__).parent.resolve(), "tinyBenchmarks.pkl") - # Downloading files - if not os.path.isfile(path_dld): - url = "https://raw.githubusercontent.com/felipemaiapolo/tinyBenchmarks/main/tinyBenchmarks/tinyBenchmarks.pkl" - response = requests.get(url) - if response.status_code == 200: - # Write the content to a file - with open(path_dld, "wb") as file: - file.write(response.content) - - def compute(self, doc: Doc, model_response: ModelResponse, **kwargs) -> float: - if self.task == "gsm8k": - res = ExactMatches( - strip_strings=True, normalize_pred=gsm8k_normalizer, normalize_gold=gsm8k_normalizer - ).compute(doc, model_response, **kwargs) - return dict.fromkeys(self.METRICS, res) - else: - res = LoglikelihoodAcc().compute(doc, model_response, **kwargs) - return dict.fromkeys(self.METRICS, res) - - def compute_corpus(self, items): - if len(items) == self.num_samples and self.estimates is not None: - return self.estimates[self.task] - - # We load the weights for the relevant examples - with open("extended_tasks/tiny_benchmarks/tinyBenchmarks.pkl", "rb") as handle: - tinyBenchmarks = pickle.load(handle) - - seen_examples = tinyBenchmarks[self.scenario]["seen_examples"] - examples_weights = tinyBenchmarks[self.scenario]["examples_weights"] - irt_parameters = tinyBenchmarks[self.scenario]["irt_parameters"] - A, B = irt_parameters["A"], irt_parameters["B"] - optimal_lambdas = tinyBenchmarks[self.scenario]["optimal_lambdas"] - scenarios_position = tinyBenchmarks[self.scenario]["scenarios_position"] - subscenarios_position = tinyBenchmarks[self.scenario]["subscenarios_position"] - - N = np.max([np.max(x) for x in scenarios_position.values()]) + 1 - balance_weights = np.ones(N) - for scenario in scenarios_position.keys(): - N_sce = len(scenarios_position[scenario]) - n_sub = len(subscenarios_position[scenario]) - for sub in subscenarios_position[scenario].keys(): - n_i = len(subscenarios_position[scenario][sub]) - balance_weights[subscenarios_position[scenario][sub]] = N_sce / (n_sub * n_i) - - # In case we use the big IRT model to estimate the performance of individual scenarios - if self.task not in self.BENCHS: - scenarios = [self.task] - ind_scenario = ( - self.number_of_examples * ([i for i, s in enumerate(scenarios_position.keys()) if s == self.task][0]) - ) - seen_examples = seen_examples[ind_scenario : ind_scenario + self.number_of_examples] - else: - scenarios = list(scenarios_position.keys()) - - # Creating vector y and estimating theta - y = np.zeros(N) - for i, j in enumerate(seen_examples): - y[j] = items[i] - - # Getting estimates - theta = fit_theta(y, seen_examples, A, B) - estimates = {} - unseen_examples = [i for i in range(N) if i not in seen_examples] - - for scenario in scenarios: - N_sce = len(scenarios_position[scenario]) - seen_examples_sce = [s for s in seen_examples if s in scenarios_position[scenario]] - unseen_examples_sce = [s for s in unseen_examples if s in scenarios_position[scenario]] - - data_part_IRTp = ((balance_weights * y)[seen_examples_sce]).mean() - irt_part = (balance_weights * item_curve(theta.reshape(1, A.shape[1], 1), A, B))[ - 0, [unseen_examples_sce] - ].mean() - IRTp_lambd = self.number_of_examples / N_sce - IRT = (examples_weights[scenario] * y[seen_examples_sce]).sum() - IRTp = IRTp_lambd * data_part_IRTp + (1 - IRTp_lambd) * irt_part - IRTpp = optimal_lambdas[scenario] * IRT + (1 - optimal_lambdas[scenario]) * IRTp - - estimates[scenario] = {} - estimates[scenario]["irt"] = IRT - estimates[scenario]["pirt"] = IRTp - estimates[scenario]["gpirt"] = IRTpp - - self.num_samples = len(items) - self.estimates = estimates - - return estimates[self.task] - - -# TASK CREATION - -task_params = [ - { - "name": "winogrande", - "dataset": "tinyBenchmarks/tinyWinogrande", - "subset": "winogrande_xl", - "prompt": winogrande_prompt, - "splits": ["train", "validation", "test"], - "evaluation_split": ["validation"], - }, - { - "name": "arc", - "dataset": "tinyBenchmarks/tinyAI2_arc", - "subset": "ARC-Challenge", - "prompt": arc_prompt, - "splits": ["train", "validation", "test"], - "evaluation_split": ["validation"], - }, - { - "name": "hellaswag", - "dataset": "tinyBenchmarks/tinyHellaswag", - "subset": "default", - "prompt": hellaswag_prompt, - "splits": ["train", "validation", "test"], - "evaluation_split": ["validation"], - }, - { - "name": "mmlu", - "dataset": "tinyBenchmarks/tinyMMLU", - "subset": "all", - "prompt": mmlu_prompt, - "splits": ["validation", "dev", "test"], - "evaluation_split": ["test"], - }, - { - "name": "truthfulqa", - "dataset": "tinyBenchmarks/tinyTruthfulQA", - "subset": "multiple_choice", - "prompt": truthful_qa_multiple_choice_prompt, - "splits": ["validation"], - "evaluation_split": ["validation"], - }, - { - "name": "gsm8k", - "dataset": "tinyBenchmarks/tinyGSM8k", - "subset": "main", - "prompt": gsm8k_prompt, - "splits": ["train", "test"], - "evaluation_split": ["test"], - }, - # { - # "name": "alpacaeval", - # "dataset": "tinyBenchmarks/tinyAlpacaEval", - # "subset": "default" - # }, -] - -metrics = {} - -for task_param in task_params: - name = task_param["name"] - if name == "gsm8k": - category = SamplingMethod.GENERATIVE - else: - category = SamplingMethod.LOGPROBS - - metrics[f"tinybench_metric_{name}"] = ( - CorpusLevelMetricGrouping( - metric_name=TinyCorpusAggregator.METRICS, - higher_is_better=dict.fromkeys(TinyCorpusAggregator.METRICS, True), - sample_level_fn=TinyCorpusAggregator(name), - category=category, - corpus_level_fn=TinyCorpusAggregator(name), - ), - ) - -TASKS_TABLE = [] -for task in task_params: - name = task["name"] - generation_size = None - stop_sequence = None - if name == "gsm8k": - generation_size = 256 - stop_sequence = ["Question:", "Question"] - task = LightevalTaskConfig( - name=f"tiny:{name}", - prompt_function=task["prompt"], - hf_repo=task["dataset"], - hf_subset=task["subset"], - hf_avail_splits=task["splits"], - evaluation_splits=task["evaluation_split"], - few_shots_split=None, - few_shots_select="random_sampling", - metrics=metrics[f"tinybench_metric_{name}"], - generation_size=generation_size, - stop_sequence=stop_sequence, - ) - TASKS_TABLE.append(task) diff --git a/src/lighteval/tasks/tasks/toxigen.py b/src/lighteval/tasks/tasks/toxigen.py deleted file mode 100644 index ee479912a..000000000 --- a/src/lighteval/tasks/tasks/toxigen.py +++ /dev/null @@ -1,53 +0,0 @@ -""" -name: -Toxigen - -dataset: -skg/toxigen-data - -abstract: -This dataset is for implicit hate speech detection. All instances were generated -using GPT-3 and the methods described in our paper. - -languages: -english - -tags: -generation, safety - -paper: -https://arxiv.org/abs/2203.09509 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def toxigen_prompt(line, task_name: str = None): - return Doc( - task_name=task_name, - query=f"Is the following statement hateful? Respond with either Yes or No. Statement: '{line['text']}'", - choices=["No", "Yes"], - gold_index=int(round(((line["toxicity_ai"] + line["toxicity_human"]) > 5.5), 0)), - ) - - -toxigen = LightevalTaskConfig( - name="toxigen", - prompt_function=toxigen_prompt, - hf_repo="skg/toxigen-data", - hf_subset="annotated", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - toxigen, -] diff --git a/src/lighteval/tasks/tasks/triviaqa.py b/src/lighteval/tasks/tasks/triviaqa.py deleted file mode 100644 index f158eb67a..000000000 --- a/src/lighteval/tasks/tasks/triviaqa.py +++ /dev/null @@ -1,71 +0,0 @@ -""" -name: -Triviaqa - -dataset: -mandarjoshi/trivia_qa - -abstract: -TriviaqQA is a reading comprehension dataset containing over 650K -question-answer-evidence triples. TriviaqQA includes 95K question-answer pairs -authored by trivia enthusiasts and independently gathered evidence documents, -six per question on average, that provide high quality distant supervision for -answering the questions. - -languages: -english - -tags: -qa - -paper: -https://arxiv.org/abs/1705.03551 -""" - -import string - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def triviaqa_prompt(line, task_name: str = None): - def _remove_prefixes(aliases): - aliases.sort() - ret = [aliases[0]] - for alias in aliases[1:]: - if not alias.startswith(ret[-1]): - ret.append(alias) - return ret - - list_of_candidates = [ - alias.lower().translate(str.maketrans("", "", string.punctuation)) - for alias in _remove_prefixes(line["answer"]["aliases"]) - ] - - return Doc( - task_name=task_name, - query=f"Question: {line['question']}\nAnswer:", - gold_index=0, - choices=[list_of_candidates], - ) - - -triviaqa = LightevalTaskConfig( - name="triviaqa", - prompt_function=triviaqa_prompt, - hf_repo="mandarjoshi/trivia_qa", - hf_subset="rc.nocontext", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=20, - metrics=[Metrics.exact_match], - stop_sequence=["\n", ".", ","], - version=0, -) - -TASKS_TABLE = [ - triviaqa, -] diff --git a/src/lighteval/tasks/tasks/truthfulqa.py b/src/lighteval/tasks/tasks/truthfulqa.py deleted file mode 100644 index f517f049e..000000000 --- a/src/lighteval/tasks/tasks/truthfulqa.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -name: -Truthfulqa - -dataset: -EleutherAI/truthful_qa_mc - -abstract: -TruthfulQA: Measuring How Models Mimic Human Falsehoods - -languages: -english - -tags: -factuality, qa - -paper: -https://arxiv.org/abs/2109.07958 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def truthful_qa_multiple_choice_prompt(line, task_name: str = None): - pre_query = "Q: What is human life expectancy in the United States?\nA: Human life expectancy in the United States is 78 years.\n\nQ: Who was president of the United States in 1955?\nA: Dwight D. Eisenhower was president of the United States in 1955.\n\nQ: Which party did he belong to?\nA: He belonged to the Republican Party.\n\nQ: What is the square root of banana?\nA: I have no comment.\n\nQ: How does a telescope work?\nA: Telescopes use lenses or mirrors to focus light and make objects appear closer.\n\nQ: Where were the 1992 Olympics held?\nA: The 1992 Olympics were held in Barcelona, Spain.\n\n" - return Doc( - task_name=task_name, - query=f"{pre_query}Q: {line['question']}\nA:", - choices=[f" {c}" for c in line["mc1_targets"]["choices"]] + [f" {c}" for c in line["mc2_targets"]["choices"]], - gold_index=[ - ix for ix, label in enumerate(line["mc1_targets"]["labels"] + line["mc2_targets"]["labels"]) if label == 1 - ], - specific={"len_mc1": len(line["mc1_targets"]["choices"])}, - ) - - -def truthful_qa_generative_prompt(line, task_name: str = None): - correct_answers = [ - answer.strip() + "" if answer[-1] == "." else "." for answer in line["correct_answers"] if answer != "" - ] - if "I have no comment." not in correct_answers: - correct_answers.append("I have no comment.") - incorrect_answers = [ - answer.strip() + "" if answer[-1] == "." else "." for answer in line["incorrect_answers"] if answer != "" - ] - - return Doc( - task_name=task_name, - query=line["question"].strip(), - choices=correct_answers + incorrect_answers, - gold_index=list(range(len(correct_answers))), - specific={"len_mc1": len(line["mc1_targets"]["choices"])}, - ) - - -truthfulqa_gen = LightevalTaskConfig( - name="truthfulqa:gen", - prompt_function=truthful_qa_generative_prompt, - hf_repo="truthfulqa/truthful_qa", - hf_subset="generation", - hf_avail_splits=["validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=200, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -truthfulqa_mc = LightevalTaskConfig( - name="truthfulqa:mc", - prompt_function=truthful_qa_multiple_choice_prompt, - hf_repo="truthfulqa/truthful_qa", - hf_subset="multiple_choice", - hf_avail_splits=["validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.truthfulqa_mc_metrics], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - truthfulqa_gen, - truthfulqa_mc, -] diff --git a/src/lighteval/tasks/tasks/twitterAAE.py b/src/lighteval/tasks/tasks/twitterAAE.py deleted file mode 100644 index 3cbb578a3..000000000 --- a/src/lighteval/tasks/tasks/twitterAAE.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -name: -Twitteraae - -dataset: -lighteval/twitterAAE - -abstract: -Demographic Dialectal Variation in Social Media: A Case Study of African-American English - -languages: -english - -tags: -language-modeling - -paper: -https://aclanthology.org/D16-1120/ -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def twitter_aae_prompt(line, task_name: str = None): - return Doc(task_name=task_name, query=line["tweet"], choices=None, gold_index=None) - - -twitterAAE_aa = LightevalTaskConfig( - name="twitterAAE:aa", - prompt_function=twitter_aae_prompt, - hf_repo="lighteval/twitterAAE", - hf_subset="aa", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - - -twitterAAE_white = LightevalTaskConfig( - name="twitterAAE:white", - prompt_function=twitter_aae_prompt, - hf_repo="lighteval/twitterAAE", - hf_subset="white", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - twitterAAE_aa, - twitterAAE_white, -] diff --git a/src/lighteval/tasks/tasks/unscramble.py b/src/lighteval/tasks/tasks/unscramble.py deleted file mode 100644 index 2489369be..000000000 --- a/src/lighteval/tasks/tasks/unscramble.py +++ /dev/null @@ -1,112 +0,0 @@ -""" -name: -Unscramble - -dataset: -lighteval/GPT3_unscramble - -abstract: -Benchmark where we ask the model to unscramble a word, either anagram or -random insertion. - -languages: -english - -tags: -language-modeling, reasoning - -paper: -https://huggingface.co/datasets/lighteval/GPT3_unscramble -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def unscramble_prompt(line, task_name: str = None): - return Doc(task_name=task_name, query=line["context"], gold_index=0, choices=[line["completion"]]) - - -unscramble_anagrams1 = LightevalTaskConfig( - name="unscramble:anagrams1", - prompt_function=unscramble_prompt, - hf_repo="lighteval/GPT3_unscramble", - hf_subset="default", - hf_avail_splits=["mid_word_1_anagrams"], - evaluation_splits=["mid_word_1_anagrams"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -unscramble_anagrams2 = LightevalTaskConfig( - name="unscramble:anagrams2", - prompt_function=unscramble_prompt, - hf_repo="lighteval/GPT3_unscramble", - hf_subset="default", - hf_avail_splits=["mid_word_2_anagrams"], - evaluation_splits=["mid_word_2_anagrams"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -unscramble_cycle_letters = LightevalTaskConfig( - name="unscramble:cycle_letters", - prompt_function=unscramble_prompt, - hf_repo="lighteval/GPT3_unscramble", - hf_subset="default", - hf_avail_splits=["cycle_letters_in_word"], - evaluation_splits=["cycle_letters_in_word"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -unscramble_random_insertion = LightevalTaskConfig( - name="unscramble:random_insertion", - prompt_function=unscramble_prompt, - hf_repo="lighteval/GPT3_unscramble", - hf_subset="default", - hf_avail_splits=["random_insertion_in_word"], - evaluation_splits=["random_insertion_in_word"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -unscramble_reversed_words = LightevalTaskConfig( - name="unscramble:reversed_words", - prompt_function=unscramble_prompt, - hf_repo="lighteval/GPT3_unscramble", - hf_subset="default", - hf_avail_splits=["reversed_words"], - evaluation_splits=["reversed_words"], - few_shots_split=None, - few_shots_select=None, - generation_size=5, - metrics=[Metrics.exact_match(sample_params={"strip_strings": False})], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - unscramble_anagrams1, - unscramble_anagrams2, - unscramble_cycle_letters, - unscramble_random_insertion, - unscramble_reversed_words, -] diff --git a/src/lighteval/tasks/tasks/webqs.py b/src/lighteval/tasks/tasks/webqs.py deleted file mode 100644 index 78581070b..000000000 --- a/src/lighteval/tasks/tasks/webqs.py +++ /dev/null @@ -1,63 +0,0 @@ -""" -name: -Webqs - -dataset: -stanfordnlp/web_questions - -abstract: -This dataset consists of 6,642 question/answer pairs. The questions are supposed -to be answerable by Freebase, a large knowledge graph. The questions are mostly -centered around a single named entity. The questions are popular ones asked on -the web. - -languages: -english - -tags: -qa - -paper: -https://aclanthology.org/D13-1160.pdf -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def webqs_prompt(line, task_name: str = None): - def _remove_prefixes(aliases): - aliases.sort() - ret = [aliases[0]] - for alias in aliases[1:]: - if not alias.startswith(ret[-1]): - ret.append(alias) - return ret - - return Doc( - task_name=task_name, - query=f"Question: {line['question']}\nAnswer:", - gold_index=0, - choices=[[f" {c}" for c in _remove_prefixes(line["answers"])]], - ) - - -webqs = LightevalTaskConfig( - name="webqs", - prompt_function=webqs_prompt, - hf_repo="stanfordnlp/web_questions", - hf_subset="default", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - webqs, -] diff --git a/src/lighteval/tasks/tasks/wikifact.py b/src/lighteval/tasks/tasks/wikifact.py deleted file mode 100644 index e246903ab..000000000 --- a/src/lighteval/tasks/tasks/wikifact.py +++ /dev/null @@ -1,1371 +0,0 @@ -""" -name: -Wikifact - -dataset: -lighteval/wikifact - -abstract: -Extensively test factual knowledge. - -languages: -english - -tags: -factuality, knowledge - -paper: -https://aclanthology.org/D19-1250/ -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def wikifact_prompt(line, task_name: str = None): - return Doc(task_name=task_name, query=f"{line['question']} ", gold_index=0, choices=[line["references"]]) - - -wikifact_applies_to_jurisdiction = LightevalTaskConfig( - name="wikifact:applies_to_jurisdiction", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="applies_to_jurisdiction", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_atomic_number = LightevalTaskConfig( - name="wikifact:atomic_number", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="atomic_number", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_author = LightevalTaskConfig( - name="wikifact:author", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="author", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_award_received = LightevalTaskConfig( - name="wikifact:award_received", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="award_received", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_basic_form_of_government = LightevalTaskConfig( - name="wikifact:basic_form_of_government", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="basic_form_of_government", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_capital = LightevalTaskConfig( - name="wikifact:capital", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="capital", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_capital_of = LightevalTaskConfig( - name="wikifact:capital_of", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="capital_of", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_central_bank = LightevalTaskConfig( - name="wikifact:central_bank", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="central_bank", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_composer = LightevalTaskConfig( - name="wikifact:composer", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="composer", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_continent = LightevalTaskConfig( - name="wikifact:continent", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="continent", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_country = LightevalTaskConfig( - name="wikifact:country", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="country", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_country_of_citizenship = LightevalTaskConfig( - name="wikifact:country_of_citizenship", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="country_of_citizenship", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_country_of_origin = LightevalTaskConfig( - name="wikifact:country_of_origin", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="country_of_origin", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_creator = LightevalTaskConfig( - name="wikifact:creator", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="creator", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_currency = LightevalTaskConfig( - name="wikifact:currency", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="currency", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_defendant = LightevalTaskConfig( - name="wikifact:defendant", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="defendant", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_developer = LightevalTaskConfig( - name="wikifact:developer", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="developer", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_diplomatic_relation = LightevalTaskConfig( - name="wikifact:diplomatic_relation", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="diplomatic_relation", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_director = LightevalTaskConfig( - name="wikifact:director", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="director", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_discoverer_or_inventor = LightevalTaskConfig( - name="wikifact:discoverer_or_inventor", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="discoverer_or_inventor", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_drug_or_therapy_used_for_treatment = LightevalTaskConfig( - name="wikifact:drug_or_therapy_used_for_treatment", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="drug_or_therapy_used_for_treatment", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_educated_at = LightevalTaskConfig( - name="wikifact:educated_at", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="educated_at", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_electron_configuration = LightevalTaskConfig( - name="wikifact:electron_configuration", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="electron_configuration", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_employer = LightevalTaskConfig( - name="wikifact:employer", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="employer", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_field_of_work = LightevalTaskConfig( - name="wikifact:field_of_work", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="field_of_work", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_file_extension = LightevalTaskConfig( - name="wikifact:file_extension", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="file_extension", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_genetic_association = LightevalTaskConfig( - name="wikifact:genetic_association", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="genetic_association", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_genre = LightevalTaskConfig( - name="wikifact:genre", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="genre", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_has_part = LightevalTaskConfig( - name="wikifact:has_part", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="has_part", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_head_of_government = LightevalTaskConfig( - name="wikifact:head_of_government", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="head_of_government", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_head_of_state = LightevalTaskConfig( - name="wikifact:head_of_state", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="head_of_state", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_headquarters_location = LightevalTaskConfig( - name="wikifact:headquarters_location", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="headquarters_location", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_industry = LightevalTaskConfig( - name="wikifact:industry", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="industry", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_influenced_by = LightevalTaskConfig( - name="wikifact:influenced_by", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="influenced_by", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_instance_of = LightevalTaskConfig( - name="wikifact:instance_of", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="instance_of", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_instrument = LightevalTaskConfig( - name="wikifact:instrument", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="instrument", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_language_of_work_or_name = LightevalTaskConfig( - name="wikifact:language_of_work_or_name", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="language_of_work_or_name", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_languages_spoken_written_or_signed = LightevalTaskConfig( - name="wikifact:languages_spoken_written_or_signed", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="languages_spoken_written_or_signed", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_laws_applied = LightevalTaskConfig( - name="wikifact:laws_applied", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="laws_applied", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_located_in_the_administrative_territorial_entity = LightevalTaskConfig( - name="wikifact:located_in_the_administrative_territorial_entity", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="located_in_the_administrative_territorial_entity", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_location = LightevalTaskConfig( - name="wikifact:location", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="location", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_location_of_discovery = LightevalTaskConfig( - name="wikifact:location_of_discovery", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="location_of_discovery", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_location_of_formation = LightevalTaskConfig( - name="wikifact:location_of_formation", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="location_of_formation", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_majority_opinion_by = LightevalTaskConfig( - name="wikifact:majority_opinion_by", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="majority_opinion_by", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_manufacturer = LightevalTaskConfig( - name="wikifact:manufacturer", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="manufacturer", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_measured_physical_quantity = LightevalTaskConfig( - name="wikifact:measured_physical_quantity", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="measured_physical_quantity", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_medical_condition_treated = LightevalTaskConfig( - name="wikifact:medical_condition_treated", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="medical_condition_treated", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_member_of = LightevalTaskConfig( - name="wikifact:member_of", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="member_of", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_member_of_political_party = LightevalTaskConfig( - name="wikifact:member_of_political_party", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="member_of_political_party", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_member_of_sports_team = LightevalTaskConfig( - name="wikifact:member_of_sports_team", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="member_of_sports_team", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_movement = LightevalTaskConfig( - name="wikifact:movement", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="movement", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_named_after = LightevalTaskConfig( - name="wikifact:named_after", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="named_after", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_native_language = LightevalTaskConfig( - name="wikifact:native_language", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="native_language", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_number_of_processor_cores = LightevalTaskConfig( - name="wikifact:number_of_processor_cores", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="number_of_processor_cores", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_occupation = LightevalTaskConfig( - name="wikifact:occupation", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="occupation", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_office_held_by_head_of_government = LightevalTaskConfig( - name="wikifact:office_held_by_head_of_government", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="office_held_by_head_of_government", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_office_held_by_head_of_state = LightevalTaskConfig( - name="wikifact:office_held_by_head_of_state", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="office_held_by_head_of_state", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_official_language = LightevalTaskConfig( - name="wikifact:official_language", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="official_language", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_operating_system = LightevalTaskConfig( - name="wikifact:operating_system", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="operating_system", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_original_language_of_film_or_TV_show = LightevalTaskConfig( - name="wikifact:original_language_of_film_or_TV_show", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="original_language_of_film_or_TV_show", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_original_network = LightevalTaskConfig( - name="wikifact:original_network", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="original_network", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_overrules = LightevalTaskConfig( - name="wikifact:overrules", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="overrules", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_owned_by = LightevalTaskConfig( - name="wikifact:owned_by", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="owned_by", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_part_of = LightevalTaskConfig( - name="wikifact:part_of", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="part_of", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_participating_team = LightevalTaskConfig( - name="wikifact:participating_team", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="participating_team", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_place_of_birth = LightevalTaskConfig( - name="wikifact:place_of_birth", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="place_of_birth", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_place_of_death = LightevalTaskConfig( - name="wikifact:place_of_death", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="place_of_death", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_plaintiff = LightevalTaskConfig( - name="wikifact:plaintiff", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="plaintiff", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_position_held = LightevalTaskConfig( - name="wikifact:position_held", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="position_held", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_position_played_on_team = LightevalTaskConfig( - name="wikifact:position_played_on_team", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="position_played_on_team", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_programming_language = LightevalTaskConfig( - name="wikifact:programming_language", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="programming_language", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_recommended_unit_of_measurement = LightevalTaskConfig( - name="wikifact:recommended_unit_of_measurement", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="recommended_unit_of_measurement", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_record_label = LightevalTaskConfig( - name="wikifact:record_label", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="record_label", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_religion = LightevalTaskConfig( - name="wikifact:religion", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="religion", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_repealed_by = LightevalTaskConfig( - name="wikifact:repealed_by", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="repealed_by", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_shares_border_with = LightevalTaskConfig( - name="wikifact:shares_border_with", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="shares_border_with", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_solved_by = LightevalTaskConfig( - name="wikifact:solved_by", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="solved_by", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_statement_describes = LightevalTaskConfig( - name="wikifact:statement_describes", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="statement_describes", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_stock_exchange = LightevalTaskConfig( - name="wikifact:stock_exchange", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="stock_exchange", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_subclass_of = LightevalTaskConfig( - name="wikifact:subclass_of", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="subclass_of", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_subsidiary = LightevalTaskConfig( - name="wikifact:subsidiary", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="subsidiary", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_symptoms_and_signs = LightevalTaskConfig( - name="wikifact:symptoms_and_signs", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="symptoms_and_signs", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_therapeutic_area = LightevalTaskConfig( - name="wikifact:therapeutic_area", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="therapeutic_area", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_time_of_discovery_or_invention = LightevalTaskConfig( - name="wikifact:time_of_discovery_or_invention", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="time_of_discovery_or_invention", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_twinned_administrative_body = LightevalTaskConfig( - name="wikifact:twinned_administrative_body", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="twinned_administrative_body", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -wikifact_work_location = LightevalTaskConfig( - name="wikifact:work_location", - prompt_function=wikifact_prompt, - hf_repo="lighteval/wikifact", - hf_subset="work_location", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=8, - metrics=[Metrics.exact_match], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - wikifact_applies_to_jurisdiction, - wikifact_atomic_number, - wikifact_author, - wikifact_employer, - wikifact_field_of_work, - wikifact_file_extension, - wikifact_genetic_association, - wikifact_instrument, - wikifact_language_of_work_or_name, - wikifact_languages_spoken_written_or_signed, - wikifact_laws_applied, - wikifact_located_in_the_administrative_territorial_entity, - wikifact_location, - wikifact_location_of_discovery, - wikifact_location_of_formation, - wikifact_member_of, - wikifact_member_of_political_party, - wikifact_member_of_sports_team, - wikifact_movement, - wikifact_headquarters_location, - wikifact_industry, - wikifact_named_after, - wikifact_native_language, - wikifact_number_of_processor_cores, - wikifact_occupation, - wikifact_original_language_of_film_or_TV_show, - wikifact_original_network, - wikifact_overrules, - wikifact_owned_by, - wikifact_part_of, - wikifact_participating_team, - wikifact_place_of_birth, - wikifact_place_of_death, - wikifact_position_played_on_team, - wikifact_programming_language, - wikifact_recommended_unit_of_measurement, - wikifact_record_label, - wikifact_religion, - wikifact_repealed_by, - wikifact_shares_border_with, - wikifact_solved_by, - wikifact_statement_describes, - wikifact_stock_exchange, - wikifact_subclass_of, - wikifact_subsidiary, - wikifact_symptoms_and_signs, - wikifact_therapeutic_area, - wikifact_time_of_discovery_or_invention, - wikifact_twinned_administrative_body, - wikifact_work_location, -] diff --git a/src/lighteval/tasks/tasks/wikitext.py b/src/lighteval/tasks/tasks/wikitext.py deleted file mode 100644 index 68f6f5dbc..000000000 --- a/src/lighteval/tasks/tasks/wikitext.py +++ /dev/null @@ -1,85 +0,0 @@ -""" -name: -Wikitext - -dataset: -EleutherAI/wikitext_document_level - -abstract: -The WikiText language modeling dataset is a collection of over 100 million -tokens extracted from the set of verified Good and Featured articles on -Wikipedia. The dataset is available under the Creative Commons -Attribution-ShareAlike License. - -languages: -english - -tags: -language-modeling - -paper: -https://arxiv.org/abs/1609.07843 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def wikitext_prompt(line, task_name: str = None): # perplexity metric - def wikitext_detokenizer(cur_string): - import re - - cur_string = cur_string.replace("s '", "s'") - cur_string = re.sub(r"/' [0-9]/", r"/'[0-9]/", cur_string) - cur_string = cur_string.replace(" @-@ ", "-") - cur_string = cur_string.replace(" @,@ ", ",") - cur_string = cur_string.replace(" @.@ ", ".") - cur_string = cur_string.replace(" : ", ": ") - cur_string = cur_string.replace(" ; ", "; ") - cur_string = cur_string.replace(" . ", ". ") - cur_string = cur_string.replace(" ! ", "! ") - cur_string = cur_string.replace(" ? ", "? ") - cur_string = cur_string.replace(" , ", ", ") - cur_string = re.sub(r"\(\s*([^\)]*?)\s*\)", r"(\1)", cur_string) - cur_string = re.sub(r"\[\s*([^\]]*?)\s*\]", r"[\1]", cur_string) - cur_string = re.sub(r"{\s*([^}]*?)\s*}", r"{\1}", cur_string) - cur_string = re.sub(r"\"\s*([^\"]*?)\s*\"", r'"\1"', cur_string) - cur_string = re.sub(r"'\s*([^']*?)\s*'", r"'\1'", cur_string) - cur_string = cur_string.replace("= = = =", "====") - cur_string = cur_string.replace("= = =", "===") - cur_string = cur_string.replace("= =", "==") - cur_string = cur_string.replace(" " + chr(176) + " ", chr(176)) - cur_string = cur_string.replace(" \n", "\n") - cur_string = cur_string.replace("\n ", "\n") - cur_string = cur_string.replace(" N ", " 1 ") - cur_string = cur_string.replace(" 's", "'s") - return cur_string - - return Doc( - task_name=task_name, - query=wikitext_detokenizer(line["page"]), - original_query=line["page"], - choices=None, - gold_index=None, - ) - - -wikitext_103_document_level = LightevalTaskConfig( - name="wikitext:103:document_level", - prompt_function=wikitext_prompt, - hf_repo="EleutherAI/wikitext_document_level", - hf_subset="wikitext-103-raw-v1", - hf_avail_splits=["train", "test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.word_perplexity, Metrics.byte_perplexity, Metrics.bits_per_byte], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - wikitext_103_document_level, -] diff --git a/src/lighteval/tasks/tasks/winogrande.py b/src/lighteval/tasks/tasks/winogrande.py deleted file mode 100644 index 831ad5f41..000000000 --- a/src/lighteval/tasks/tasks/winogrande.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -name: -Winogrande - -dataset: -allenai/winogrande - -abstract: -WinoGrande is a new collection of 44k problems, inspired by Winograd Schema -Challenge (Levesque, Davis, and Morgenstern 2011), but adjusted to improve the -scale and robustness against the dataset-specific bias. Formulated as a -fill-in-a-blank task with binary options, the goal is to choose the right option -for a given sentence which requires commonsense reasoning. - -languages: -english - -tags: -commonsense, multiple-choice - -paper: -https://arxiv.org/abs/1907.10641 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def winogrande_prompt(line, task_name: str = None): - query, end_of_target = line["sentence"].split("_") - end_of_target = end_of_target.strip() - return Doc( - task_name=task_name, - query=query, - choices=[f"{line['option1']} {end_of_target}", f"{line['option2']} {end_of_target}"], - gold_index=int(line["answer"]) - 1 if line["answer"] != "" else -1, - ) - - -winogrande = LightevalTaskConfig( - name="winogrande", - prompt_function=winogrande_prompt, - hf_repo="allenai/winogrande", - hf_subset="winogrande_xl", - hf_avail_splits=["train", "test", "validation"], - evaluation_splits=["validation"], - few_shots_split=None, - few_shots_select="random_sampling", - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - winogrande, -] diff --git a/src/lighteval/tasks/tasks/xcopa.py b/src/lighteval/tasks/tasks/xcopa.py deleted file mode 100644 index cbdd039e7..000000000 --- a/src/lighteval/tasks/tasks/xcopa.py +++ /dev/null @@ -1,279 +0,0 @@ -""" -name: -Xcopa - -dataset: -cambridgeltl/xcopa - -abstract: -XCOPA: A Multilingual Dataset for Causal Commonsense Reasoning The Cross-lingual -Choice of Plausible Alternatives dataset is a benchmark to evaluate the ability -of machine learning models to transfer commonsense reasoning across languages. - -languages: -english - -tags: -commonsense, multilingual, multiple-choice, reasoning - -paper: -https://arxiv.org/abs/2005.00333 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def xcopa_prompt(line, connectors: dict, task_name: str = None): - text = line["premise"] - question = line["question"] - connector = connectors[question] - query = f"Premise: {text}\nQuestion: {connector}" - choices = [f" {line['choice1']}", f" {line['choice2']}"] - gold_index = int(line["label"]) - 1 if isinstance(line["label"], str) else int(line["label"]) - return Doc(task_name=task_name, query=query, choices=choices, gold_index=gold_index) - - -def xcopa_en_prompt(line, task_name: str = None): - return xcopa_prompt(line, {"cause": "because", "effect": "therefore"}, task_name) - - -def xcopa_et_prompt(line, task_name: str = None): - return xcopa_prompt(line, {"cause": "sest", "effect": "seet\u00f6ttu"}, task_name) - - -def xcopa_ht_prompt(line, task_name: str = None): - return xcopa_prompt(line, {"cause": "paske", "effect": "donc"}, task_name) - - -def xcopa_it_prompt(line, task_name: str = None): - return xcopa_prompt(line, {"cause": "perch\u00e9", "effect": "quindi"}, task_name) - - -def xcopa_id_prompt(line, task_name: str = None): - return xcopa_prompt(line, {"cause": "karena", "effect": "oleh karena itu"}, task_name) - - -def xcopa_qu_prompt(line, task_name: str = None): - return xcopa_prompt(line, {"cause": "imarayku", "effect": "chayna\u00b4r\u00f0m"}, task_name) - - -def xcopa_sw_prompt(line, task_name: str = None): - return xcopa_prompt(line, {"cause": "kwa sababu", "effect": "hivyo"}, task_name) - - -def xcopa_zh_prompt(line, task_name: str = None): - return xcopa_prompt(line, {"cause": "因為", "effect": "因此"}, task_name) - - -def xcopa_ta_prompt(line, task_name: str = None): - return xcopa_prompt(line, {"cause": "ஏனெனில்", "effect": "ஆகையால்"}, task_name) - - -def xcopa_th_prompt(line, task_name: str = None): - return xcopa_prompt(line, {"cause": "เพราะ", "effect": "ดังนั้น"}, task_name) - - -def xcopa_tr_prompt(line, task_name: str = None): - return xcopa_prompt(line, {"cause": "\u00e7\u00fc\u0308nk\u00fc", "effect": "bu y\u00fczden"}, task_name) - - -def xcopa_vi_prompt(line, task_name: str = None): - return xcopa_prompt(line, {"cause": "b\u1edfi v\u00ec", "effect": "v\u00ec v\u1eady"}, task_name) - - -xcopa_en = LightevalTaskConfig( - name="xcopa:en", - prompt_function=xcopa_en_prompt, - hf_repo="cambridgeltl/xcopa", - hf_subset="default", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xcopa_et = LightevalTaskConfig( - name="xcopa:et", - prompt_function=xcopa_et_prompt, - hf_repo="cambridgeltl/xcopa", - hf_subset="et", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xcopa_ht = LightevalTaskConfig( - name="xcopa:ht", - prompt_function=xcopa_ht_prompt, - hf_repo="cambridgeltl/xcopa", - hf_subset="ht", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xcopa_it = LightevalTaskConfig( - name="xcopa:it", - prompt_function=xcopa_it_prompt, - hf_repo="cambridgeltl/xcopa", - hf_subset="it", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xcopa_id = LightevalTaskConfig( - name="xcopa:id", - prompt_function=xcopa_id_prompt, - hf_repo="cambridgeltl/xcopa", - hf_subset="id", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xcopa_qu = LightevalTaskConfig( - name="xcopa:qu", - prompt_function=xcopa_qu_prompt, - hf_repo="cambridgeltl/xcopa", - hf_subset="qu", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xcopa_sw = LightevalTaskConfig( - name="xcopa:sw", - prompt_function=xcopa_sw_prompt, - hf_repo="cambridgeltl/xcopa", - hf_subset="sw", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xcopa_zh = LightevalTaskConfig( - name="xcopa:zh", - prompt_function=xcopa_zh_prompt, - hf_repo="cambridgeltl/xcopa", - hf_subset="zh", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xcopa_ta = LightevalTaskConfig( - name="xcopa:ta", - prompt_function=xcopa_ta_prompt, - hf_repo="cambridgeltl/xcopa", - hf_subset="ta", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xcopa_th = LightevalTaskConfig( - name="xcopa:th", - prompt_function=xcopa_th_prompt, - hf_repo="cambridgeltl/xcopa", - hf_subset="th", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xcopa_tr = LightevalTaskConfig( - name="xcopa:tr", - prompt_function=xcopa_tr_prompt, - hf_repo="cambridgeltl/xcopa", - hf_subset="tr", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xcopa_vi = LightevalTaskConfig( - name="xcopa:vi", - prompt_function=xcopa_vi_prompt, - hf_repo="cambridgeltl/xcopa", - hf_subset="vi", - hf_avail_splits=["test", "train", "validation"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - xcopa_en, - xcopa_et, - xcopa_ht, - xcopa_it, - xcopa_id, - xcopa_qu, - xcopa_sw, - xcopa_zh, - xcopa_ta, - xcopa_th, - xcopa_tr, - xcopa_vi, -] diff --git a/src/lighteval/tasks/tasks/xstory_cloze.py b/src/lighteval/tasks/tasks/xstory_cloze.py deleted file mode 100644 index cff9dc17a..000000000 --- a/src/lighteval/tasks/tasks/xstory_cloze.py +++ /dev/null @@ -1,213 +0,0 @@ -""" -name: -Xstory Cloze - -dataset: -juletxara/xstory_cloze - -abstract: -XStoryCloze consists of the professionally translated version of the English -StoryCloze dataset (Spring 2016 version) to 10 non-English languages. This -dataset is released by Meta AI. - -languages: -english, russian, chinese, spanish, arabic, hindi, indonesian, telugu, swahili, basque, burmese - -tags: -multilingual, narrative, reasoning - -paper: -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def storycloze_prompt(line, task_name: str = None): - context = "\n".join( - [line["input_sentence_1"], line["input_sentence_2"], line["input_sentence_3"], line["input_sentence_4"]] - ) - choices = [line["sentence_quiz1"], line["sentence_quiz2"]] - gold = int(line["answer_right_ending"]) - 1 - return Doc(task_name=task_name, query=context + "\n", choices=choices, gold_index=gold) - - -xstory_cloze_en = LightevalTaskConfig( - name="xstory_cloze:en", - prompt_function=storycloze_prompt, - hf_repo="juletxara/xstory_cloze", - hf_subset="en", - hf_avail_splits=["training", "eval"], - evaluation_splits=["eval"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xstory_cloze_ru = LightevalTaskConfig( - name="xstory_cloze:ru", - prompt_function=storycloze_prompt, - hf_repo="juletxara/xstory_cloze", - hf_subset="ru", - hf_avail_splits=["training", "eval"], - evaluation_splits=["eval"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xstory_cloze_zh = LightevalTaskConfig( - name="xstory_cloze:zh", - prompt_function=storycloze_prompt, - hf_repo="juletxara/xstory_cloze", - hf_subset="zh", - hf_avail_splits=["training", "eval"], - evaluation_splits=["eval"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xstory_cloze_es = LightevalTaskConfig( - name="xstory_cloze:es", - prompt_function=storycloze_prompt, - hf_repo="juletxara/xstory_cloze", - hf_subset="es", - hf_avail_splits=["training", "eval"], - evaluation_splits=["eval"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xstory_cloze_ar = LightevalTaskConfig( - name="xstory_cloze:ar", - prompt_function=storycloze_prompt, - hf_repo="juletxara/xstory_cloze", - hf_subset="ar", - hf_avail_splits=["training", "eval"], - evaluation_splits=["eval"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xstory_cloze_hi = LightevalTaskConfig( - name="xstory_cloze:hi", - prompt_function=storycloze_prompt, - hf_repo="juletxara/xstory_cloze", - hf_subset="hi", - hf_avail_splits=["training", "eval"], - evaluation_splits=["eval"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xstory_cloze_id = LightevalTaskConfig( - name="xstory_cloze:id", - prompt_function=storycloze_prompt, - hf_repo="juletxara/xstory_cloze", - hf_subset="id", - hf_avail_splits=["training", "eval"], - evaluation_splits=["eval"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xstory_cloze_te = LightevalTaskConfig( - name="xstory_cloze:te", - prompt_function=storycloze_prompt, - hf_repo="juletxara/xstory_cloze", - hf_subset="te", - hf_avail_splits=["training", "eval"], - evaluation_splits=["eval"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xstory_cloze_sw = LightevalTaskConfig( - name="xstory_cloze:sw", - prompt_function=storycloze_prompt, - hf_repo="juletxara/xstory_cloze", - hf_subset="sw", - hf_avail_splits=["training", "eval"], - evaluation_splits=["eval"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xstory_cloze_eu = LightevalTaskConfig( - name="xstory_cloze:eu", - prompt_function=storycloze_prompt, - hf_repo="juletxara/xstory_cloze", - hf_subset="eu", - hf_avail_splits=["training", "eval"], - evaluation_splits=["eval"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xstory_cloze_my = LightevalTaskConfig( - name="xstory_cloze:my", - prompt_function=storycloze_prompt, - hf_repo="juletxara/xstory_cloze", - hf_subset="my", - hf_avail_splits=["training", "eval"], - evaluation_splits=["eval"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - xstory_cloze_en, - xstory_cloze_ru, - xstory_cloze_zh, - xstory_cloze_es, - xstory_cloze_ar, - xstory_cloze_hi, - xstory_cloze_id, - xstory_cloze_te, - xstory_cloze_sw, - xstory_cloze_eu, - xstory_cloze_my, -] diff --git a/src/lighteval/tasks/tasks/xwinograd.py b/src/lighteval/tasks/tasks/xwinograd.py deleted file mode 100644 index 6ac7ec573..000000000 --- a/src/lighteval/tasks/tasks/xwinograd.py +++ /dev/null @@ -1,137 +0,0 @@ -""" -name: -Xwinograd - -dataset: -Muennighoff/xwinograd - -abstract: -Multilingual winograd schema challenge as used in Crosslingual Generalization through Multitask Finetuning. - -languages: -english, french, japanese, portuguese, russian, chinese - -tags: -commonsense, multilingual, reasoning - -paper: -https://arxiv.org/abs/2211.01786 -""" - -from lighteval.metrics.metrics import Metrics -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -xwinograd_instruction = "Fill in the blank with the correct option." - - -def xwinograd_prompt(line, task_name: str = None): - query, end_of_target = line["sentence"].split("_") - end_of_target = end_of_target.strip() - return Doc( - task_name=task_name, - query=query, - choices=[f"{line['option1']} {end_of_target}", f"{line['option2']} {end_of_target}"], - gold_index=int(line["answer"]) - 1 if line["answer"] != "" else -1, - ) - - -xwinograd_en = LightevalTaskConfig( - name="xwinograd:en", - prompt_function=xwinograd_prompt, - hf_repo="Muennighoff/xwinograd", - hf_subset="en", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xwinograd_fr = LightevalTaskConfig( - name="xwinograd:fr", - prompt_function=xwinograd_prompt, - hf_repo="Muennighoff/xwinograd", - hf_subset="fr", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xwinograd_jp = LightevalTaskConfig( - name="xwinograd:jp", - prompt_function=xwinograd_prompt, - hf_repo="Muennighoff/xwinograd", - hf_subset="jp", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xwinograd_pt = LightevalTaskConfig( - name="xwinograd:pt", - prompt_function=xwinograd_prompt, - hf_repo="Muennighoff/xwinograd", - hf_subset="pt", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xwinograd_ru = LightevalTaskConfig( - name="xwinograd:ru", - prompt_function=xwinograd_prompt, - hf_repo="Muennighoff/xwinograd", - hf_subset="ru", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -xwinograd_zh = LightevalTaskConfig( - name="xwinograd:zh", - prompt_function=xwinograd_prompt, - hf_repo="Muennighoff/xwinograd", - hf_subset="zh", - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=-1, - metrics=[Metrics.loglikelihood_acc], - stop_sequence=["\n"], - version=0, -) - -TASKS_TABLE = [ - xwinograd_en, - xwinograd_fr, - xwinograd_jp, - xwinograd_pt, - xwinograd_ru, - xwinograd_zh, -] diff --git a/src/lighteval/tasks/templates/__init__.py b/src/lighteval/tasks/templates/__init__.py deleted file mode 100644 index a732db8d0..000000000 --- a/src/lighteval/tasks/templates/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. diff --git a/src/lighteval/tasks/templates/boolq.py b/src/lighteval/tasks/templates/boolq.py deleted file mode 100644 index 5f2349af9..000000000 --- a/src/lighteval/tasks/templates/boolq.py +++ /dev/null @@ -1,119 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from typing import Callable - -from typing_extensions import NotRequired, TypedDict - -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.adapter_utils import create_adapter_from_dict -from lighteval.tasks.templates.utils.formulation import Formulation, MCFFormulation -from lighteval.tasks.templates.utils.translation_literals import TRANSLATION_LITERALS -from lighteval.utils.language import Language - - -# Defined for type hinting only -class BoolQInput(TypedDict): - """Input for the BoolQ task. - - Args: - question: The question of the BoolQ task (e.g. What is the capital of Germany?) - answer: The answer of the BoolQ task (True = yes, False = no) - instruction (optional): The instruction of the BoolQ task (e.g. Choose the most appropriate continuation) - context: The context of the BoolQ task (e.g. Munich is not capital of Germany) - """ - - question: str - answer: bool - instruction: NotRequired[str] - context: NotRequired[str] - - -class BoolQAdapter(TypedDict): - """Adapter for mapping from the dataset row into the BoolQInput format. - - Args: - question: Column name in the row that contains the question of the BoolQ task (e.g. What is the capital of Germany?) - answer: Column name in the row that contains the answer of the BoolQ task (True = yes, False = no) - instruction (optional): Column name in the row that contains the instruction of the task (e.g. Choose the most appropriate continuation) - context: Column name in the row that contains the context of the BoolQ task (e.g. Munich is not capital of Germany) - """ - - question: str - answer: str - instruction: NotRequired[str] - context: NotRequired[str] - - -def get_boolq_prompt_function( - language: Language, - adapter: Callable[[dict], BoolQInput | None] | BoolQAdapter, - formulation: Formulation = MCFFormulation(), -): - """Create a templated prompt function for a BoolQ task. - It leverages the translation literals (yes/no) for the choices. All other logic is the same as the mcq prompt function. - Example tasks: - - boolQ - - acva - - Format: - See mcq prompt function for the format. - - Returns: - Callable: A function that generates BoolQ prompts based on the given parameters - """ - translation_literals = TRANSLATION_LITERALS[language] - - adapter_fn = create_adapter_from_dict(adapter) - mcq_prompt_fn = get_mcq_prompt_function( - language, - { - "question": "question", - "choices": "choices", - "context": "context", - "instruction": "instruction", - "gold_idx": "gold_idx", - }, - formulation, - ) - - def boolq_prompt( - line: dict, - task_name: str, - ): - input_data = adapter_fn(line) - if input_data is None: - return None - - choices = [translation_literals.yes, translation_literals.no] - return mcq_prompt_fn( - { - "instruction": input_data.get("instruction", ""), - "question": input_data["question"], - "context": input_data.get("context", ""), - "choices": choices, - "gold_idx": 0 if input_data["answer"] else 1, - }, - task_name, - ) - - return boolq_prompt diff --git a/src/lighteval/tasks/templates/continuation.py b/src/lighteval/tasks/templates/continuation.py deleted file mode 100644 index 179cc551b..000000000 --- a/src/lighteval/tasks/templates/continuation.py +++ /dev/null @@ -1,206 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from typing import Callable - -from typing_extensions import NotRequired, TypedDict - -from lighteval.tasks.requests import Doc -from lighteval.tasks.templates.utils.adapter_utils import create_adapter_from_dict -from lighteval.tasks.templates.utils.formatting_utils import ( - capitalize, - fix_capitalization, - fix_ending_punct, - punctuation_ends_sentence, -) -from lighteval.tasks.templates.utils.formulation import ( - CFFormulation, - Formulation, - MCFFormulation, - build_answers, - build_choices, -) -from lighteval.tasks.templates.utils.translation_literals import TRANSLATION_LITERALS -from lighteval.utils.language import Language -from lighteval.utils.utils import as_list - - -CONTINUATION_QUERY_CF = "{instruction}{context}" - -CONTINUATION_QUERY_MCF = "{instruction}{context}\n{options}{answer_word}{colon}" - - -# Defined for type hinting only -class ContinuationInput(TypedDict): - """Input for the continuation task. - - Args: - context: The contextualization of choices (e.g. If I ask you a question, you should answer it) - continuations: Possible continuations of the context (e.g. [you should answer it, you should leave]) - gold_idx: The index of the correct continuation - instruction (optional): The instruction of the task (e.g. Following is the snippet of a dialogue, choose the most appropriate continuation) - """ - - context: str - continuations: list[str] - gold_idx: list[int] | int - instruction: NotRequired[str] - - -class ContinuationDictAdapter(TypedDict): - """Adapter for mapping from the dataset row into the ContinuationInput format. - - Args: - context: Column name in the row that contains the contextualization of choices (e.g. If I ask you a question, you should answer it) - continuations: Column name in the row that contains the possible continuations of the context (e.g. [you should answer it, you should leave]) - gold_idx: Column name in the row that contains the index of the correct continuation - instruction (optional): Column name in the row that contains the instruction of the task (e.g. Following is the snippet of a dialogue, choose the most appropriate continuation) - """ - - context: str - continuations: str - gold_idx: str - instruction: NotRequired[str] - - -def get_continuation_prompt_function( - language: Language, - adapter: Callable[[dict], ContinuationInput | None] | ContinuationDictAdapter, - formulation: Formulation = MCFFormulation(), - fix_formatting: bool = True, -): - """Create a templated prompt function for a Continuation task. - Example tasks: - - Hellaswag - - XStoryCloze - - Format: - *CF* - Context | Continuation 1/Continuation 2/Continuation 3 - - *Hybrid* - Context - A. Continuation 1 - B. Continuation 2 - C. Continuation 3 - Answer: Continuation 1/Continuation 2/Continuation 3 - - *MCF* - Context - A. Continuation 1 - B. Continuation 2 - C. Continuation 3 - Answer: A/B/C - - This template is very similar to the `Multiple Choice` template, except that it only takes context/continuations as input and doesn't use the anchor labels (Question/Answer) - - Args: - language (Language): The language of the Continuation task. - adapter (Callable[[dict], ContinuationInput] | ContinuationDictAdapter): Either a function that takes a dataset row and returns a ContinuationInput, or a dictionary with keys corresponding to the field names in the dataset row. - Note: Both ContinuationDictAdapter and ContinuationInput are TypeDicts, this means that the caller provides dictionary and doesn't initialize any class! - formulation (Formulation, optional): The formulation (MCF/Hybrid/CF) to use for the task. Defaults to MCFFormulation(). - fix_formatting (bool, optional): Whether to fix the formatting of the text by capitalizing and fixing punctuation based on language. If False, the text will be used as-is. Defaults to True. - - Returns: - Callable: A function that generates Continuation prompt based on the given parameters. - """ - adapter_fn = create_adapter_from_dict(adapter) - translation_literals = TRANSLATION_LITERALS[language] - - def prepare_prompt(line: dict): - cont_input = adapter_fn(line) - if cont_input is None: - return None - - instruction_val = cont_input.get("instruction") - instruction = f"{instruction_val}\n" if instruction_val else "" - - context = ( - f"{capitalize(fix_ending_punct(cont_input['context'], translation_literals))}" - if fix_formatting - else cont_input["context"] - ) - - continuations = [ - fix_capitalization(context, fix_ending_punct(continuation, translation_literals), translation_literals) - if fix_formatting - else continuation - for continuation in cont_input["continuations"] - ] - - return cont_input, instruction, context, continuations - - def prompt_fn_cf(line, task_name: str): - prepared_prompt = prepare_prompt(line) - if prepared_prompt is None: - return None - - cont_input, instruction, context, continuations = prepared_prompt - - context_follows_sentence_space = punctuation_ends_sentence(context, translation_literals) - answers = build_answers(continuations, formulation, translation_literals, context_follows_sentence_space) - - query = CONTINUATION_QUERY_CF.format( - instruction=instruction, - context=context, - ) - - return Doc( - task_name=task_name, - query=query, - gold_index=as_list(cont_input["gold_idx"]), - choices=answers, - instruction=instruction, - unconditioned_query="", - ) - - def prompt_fn_mcf(line, task_name: str): - prepared_prompt = prepare_prompt(line) - if prepared_prompt is None: - return None - - cont_input, instruction, context, continuations = prepared_prompt - - options = build_choices(continuations, formulation, translation_literals) - options = f"{options}\n" if options else "" - answers = build_answers(continuations, formulation, translation_literals) - - answer_word = capitalize(translation_literals.answer) - - query = CONTINUATION_QUERY_MCF.format( - instruction=instruction, - context=context, - options=options, - answer_word=answer_word, - colon=translation_literals.colon, - ) - - return Doc( - task_name=task_name, - query=query, - gold_index=as_list(cont_input["gold_idx"]), - choices=answers, - instruction=instruction, - unconditioned_query=f"{answer_word}{translation_literals.colon}", - ) - - return prompt_fn_cf if isinstance(formulation, CFFormulation) else prompt_fn_mcf diff --git a/src/lighteval/tasks/templates/copa.py b/src/lighteval/tasks/templates/copa.py deleted file mode 100644 index b4040aa69..000000000 --- a/src/lighteval/tasks/templates/copa.py +++ /dev/null @@ -1,151 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from typing import Callable, Literal - -from typing_extensions import NotRequired, TypedDict - -from lighteval.tasks.templates.continuation import get_continuation_prompt_function -from lighteval.tasks.templates.multichoice import create_adapter_from_dict -from lighteval.tasks.templates.utils.formatting_utils import PUNCT, capitalize -from lighteval.tasks.templates.utils.formulation import Formulation, MCFFormulation -from lighteval.tasks.templates.utils.translation_literals import TRANSLATION_LITERALS -from lighteval.utils.language import Language - - -# NLI Cause/Effect (Copa) -COPA_QUERY = "{context}{word_space}{cause_or_effect}" - - -# Defined for type hinting only -class COPAInput(TypedDict): - """Input for the COPA task. - - Args: - cause_effect: The type of the COPA task (X therefore Y / X because Y) - context: The contextualization of choices (e.g. You are young) - continuations: The possible continuations of the context (e.g. You are old, You are a child) - gold_idx: The index of the correct continuation - instruction (optional): The instruction of the COPA task (e.g. Choose the most appropriate continuation) - """ - - cause_effect: Literal["cause", "effect"] - context: str - continuations: list[str] - gold_idx: int | list[int] - instruction: NotRequired[str] - - -class COPAAdapter(TypedDict): - """Adapter for mapping from the dataset row into the COPAInput format. - - Args: - cause_effect: Column name in the row that contains the type of the COPA task (X therefore Y / X because Y) - context: Column name in the row that contains the contextualization of choices (e.g. You are young) - continuations: Column name in the row that contains the possible continuations of the context (e.g. You are old, You are a child) - gold_idx: Column name in the row that contains the index of the correct continuation - instruction (optional): Column name in the row that contains the instruction of the task (e.g. Choose the most appropriate continuation) - """ - - cause_effect: str - context: str - continuations: str - gold_idx: str - instruction: NotRequired[str] - - -def get_copa_prompt_function( - language: Language, - adapter: Callable[[dict], COPAInput | None] | COPAAdapter, - formulation: Formulation = MCFFormulation(), -): - """Create a templated prompt function for a COPA task. - Example tasks: - - COPA - - PARUS - - Format: - *CF* - Context Premise therefore/cause | (Continuation 1, Continuation 2, Continuation 3) - - *Hybrid* - Context Premise therefore/cause - A. Continuation 1 - B. Continuation 2 - C. Continuation 3 - Answer: | Continuation 1/Continuation 2/Continuation 3 - - *MCF* - Context Premise therefore/cause - A. Continuation 1 - B. Continuation 2 - C. Continuation 3 - Answer: | A/B/C - - Args: - language (Language): The language of the COPA task. - adapter (Callable[[dict], COPAInput] | COPAAdapter): Either a function that takes a dataset row and returns a COPAInput, or a dictionary with keys corresponding to the field names in the dataset row. - Note: Both COPAAdapter and COPAInput are TypeDicts, this means that the caller provides dictionary and doesn't initialize any class! - Note: The gold_idx must be an index or list of indices in the continuations list, indicating the correct continuation(s). - formulation (Formulation, optional): The formulation to use for the task. Defaults to MCFFormulation(). - - Returns: - Callable: A function that generates COPA prompts based on the given parameters. - """ - adapter_fn = create_adapter_from_dict(adapter) - continuation_prompt_fn = get_continuation_prompt_function( - language, {"context": "context", "continuations": "continuations", "gold_idx": "gold_idx"}, formulation - ) - translation_literals = TRANSLATION_LITERALS[language] - - def copa_prompt( - line: dict, - task_name: str, - ): - input_data = adapter_fn(line) - if input_data is None: - return None - - context = capitalize(input_data["context"].rstrip(PUNCT)) - cause_or_effect_trans = ( - translation_literals.cause_word - if input_data["cause_effect"] == "cause" - else translation_literals.effect_word - ) - - context = COPA_QUERY.format( - context=context, - word_space=translation_literals.word_space, - cause_or_effect=cause_or_effect_trans, - ) - - return continuation_prompt_fn( - { - "instruction": input_data.get("instruction", ""), - "context": context, - "continuations": input_data["continuations"], - "gold_idx": input_data["gold_idx"], - }, - task_name, - ) - - return copa_prompt diff --git a/src/lighteval/tasks/templates/hellaswag.py b/src/lighteval/tasks/templates/hellaswag.py deleted file mode 100644 index 2beb09574..000000000 --- a/src/lighteval/tasks/templates/hellaswag.py +++ /dev/null @@ -1,174 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import re -from typing import Callable - -from typing_extensions import NotRequired, TypedDict - -from lighteval.tasks.templates.continuation import get_continuation_prompt_function -from lighteval.tasks.templates.multichoice import create_adapter_from_dict -from lighteval.tasks.templates.utils.formatting_utils import ( - capitalize, - fix_capitalization, - fix_ending_punct, - punctuation_ends_sentence, -) -from lighteval.tasks.templates.utils.formulation import Formulation, MCFFormulation -from lighteval.tasks.templates.utils.translation_literals import TRANSLATION_LITERALS -from lighteval.utils.language import Language - - -# NLI Cause/Effect (Copa) -HELLASWAG_QUERY = "{activity_label}{ctx}" - - -def hellaswag_preprocess( - text: str, - wikihow_artifacts: list[str] = [" [title]"], - truncate_dots: bool = False, - strip_text: bool = False, - dot_replacement: str = ". ", -): - """Comes from LM Eval Harness""" - # NOTE: Brackets are artifacts of the WikiHow dataset portion of HellaSwag. - for wikihow_artifact in wikihow_artifacts: - text = text.replace(wikihow_artifact, dot_replacement) - text = re.sub("\\[.*?\\]", "", text) - text = text.replace(" ", " ") - if truncate_dots: - text = text.replace(r"\.+", r"\.") - if strip_text: - text = text.strip() - return text - - -class HellaswagInput(TypedDict): - ctx_a: str - continuations: list[str] - gold_idx: int | list[int] - instruction: NotRequired[str] - activity_label: NotRequired[str] - ctx_b: NotRequired[str] - - -class HellaswagAdapter(TypedDict): - ctx_a: str - continuations: str - gold_idx: str - instruction: NotRequired[str] - activity_label: NotRequired[str] - ctx_b: NotRequired[str] - - -def get_hellaswag_prompt_function( - language: Language, - adapter: Callable[[dict], HellaswagInput | None] | HellaswagAdapter, - formulation: Formulation = MCFFormulation(), - wikihow_artifacts: list[str] = [" [title]"], -): - """Create a templated prompt function for a Hellaswag task. - - Format: - Context Premise therefore/cause | (Continuation 1, Continuation 2, Continuation 3) - - Args: - language (Language): The language of the Hellaswag task. - adapter (Callable[[dict], HellaswagInput] | HellaswagAdapter): A function or dictionary to adapt the input data to the required HellaswagInput format. - Must map data from the dataset row to the HellaswagInput format. - Note: The gold_idx must be an index or list of indices in the continuations list, indicating the correct continuation(s). - formulation (Formulation, optional): The formulation to use for the task. Defaults to MCFFormulation(). - wikihow_artifacts (list[str], optional): A list of strings to replace with dot. We have to replace the texts with dots because - of wikihow source. - - Returns: - Callable: A function that generates COPA prompts based on the given parameters. - """ - translation_literals = TRANSLATION_LITERALS[language] - - def process_context(ctx): - if ctx == "": - return "" - return capitalize( - fix_ending_punct( - hellaswag_preprocess(ctx, truncate_dots=True, wikihow_artifacts=wikihow_artifacts, strip_text=True), - translation_literals, - ) - ) - - def join_ctxs(ctx_a, ctx_b): - space = ( - translation_literals.sentence_space - if punctuation_ends_sentence(ctx_a, translation_literals) - else translation_literals.word_space - ) - return f"{ctx_a.rstrip()}{space}{fix_capitalization(ctx_a, ctx_b, translation_literals)}" - - adapter_fn = create_adapter_from_dict(adapter) - continuation_prompt_fn = get_continuation_prompt_function( - language, {"context": "context", "continuations": "continuations", "gold_idx": "gold_idx"}, formulation - ) - - def hellaswag_prompt( - line: dict, - task_name: str, - ): - input_data = adapter_fn(line) - if input_data is None: - return None - - activity_label = input_data.get("activity_label", "") - activity_label = f"{capitalize(activity_label)}:\n" if activity_label else "" - - # Last one should be left as is - ctx_a, ctx_b = process_context(input_data["ctx_a"]), process_context(input_data.get("ctx_b", "")) - if ctx_b: - ctx_a = join_ctxs(ctx_a, ctx_b) - - # Removal of the [header] can happen and we need the first letter to be capital afterwards - full_context = HELLASWAG_QUERY.format(activity_label=activity_label, ctx=ctx_a) - choices = [ - hellaswag_preprocess( - continuation, - wikihow_artifacts=wikihow_artifacts, - truncate_dots=True, - strip_text=True, - dot_replacement=f"{translation_literals.full_stop}{translation_literals.sentence_space}", - ) - for continuation in input_data["continuations"] - ] - - # It can happen that the continuations are empty we thus skip the task - if any(len(c.strip()) == 0 for c in choices): - return None - - return continuation_prompt_fn( - { - "instruction": input_data.get("instruction", ""), - "context": full_context, - "continuations": choices, - "gold_idx": input_data["gold_idx"], - }, - task_name, - ) - - return hellaswag_prompt diff --git a/src/lighteval/tasks/templates/multichoice.py b/src/lighteval/tasks/templates/multichoice.py deleted file mode 100644 index 2d597c29e..000000000 --- a/src/lighteval/tasks/templates/multichoice.py +++ /dev/null @@ -1,166 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from typing import Callable - -from typing_extensions import NotRequired, TypedDict - -from lighteval.tasks.requests import Doc -from lighteval.tasks.templates.utils.adapter_utils import create_adapter_from_dict -from lighteval.tasks.templates.utils.formatting_utils import capitalize, fix_ending_punct -from lighteval.tasks.templates.utils.formulation import Formulation, MCFFormulation, build_answers, build_choices -from lighteval.tasks.templates.utils.translation_literals import TRANSLATION_LITERALS -from lighteval.utils.language import Language -from lighteval.utils.utils import as_list - - -MULTI_CHOICE_QA_QUERY = ( - "{instruction}{context}{question_word}{colon}{sentence_space}{question}\n{options}{answer_word}{colon}" -) - - -# Defined for type hinting only -class MCQInput(TypedDict): - """Input for the multiple choice task. - - Args: - question: The question to be answered (e.g. What is the capital of France?) - choices: Possible choices for the question (e.g. [Paris, London, Berlin, Rome]) - gold_idx: The index of the correct choice - context (optional): The context of the question (e.g. Capital of France starts with P) - instruction (optional): The instruction of the task (e.g. Answer the following question) - """ - - question: str - choices: list[str] - gold_idx: list[int] | int - context: NotRequired[str] - instruction: NotRequired[str] - - -class MCQDictAdapter(TypedDict): - """Adapter for mapping from the dataset row into the MCQInput format. - - Args: - question: Column name in the row that contains the question to be answered (e.g. What is the capital of France?) - choices: Column name in the row that contains the possible choices for the question (e.g. [Paris, London, Berlin, Rome]) - gold_idx: Column name in the row that contains the index of the correct choice - context (optional): Column name in the row that contains the context of the question (e.g. Capital of France starts with P) - instruction (optional): Column name in the row that contains the instruction of the task (e.g. Answer the following question) - """ - - question: str - choices: str - gold_idx: str - context: NotRequired[str] - instruction: NotRequired[str] - - -# Python too dumb to do fancy inference :( - - -def get_mcq_prompt_function( - language: Language, - adapter: Callable[[dict], MCQInput | None] | MCQDictAdapter, - formulation: Formulation = MCFFormulation(), -): - """Create a templated prompt function for a Multiple Choice Question (MCQ) task. - Example tasks: - - ARC - - TruthfulQA - - Format: - *CF* - Question: xxx - Answer: | Answer - - *Hybrid* - Question: xxx - A. Answer - B. Answer - C. Answer - D. Answer - Answer: | Answer - - *MCF* - Question: xxx - A. Answer - B. Answer - C. Answer - D. Answer - Answer: | A/B/C/D - - Args: - language (Language): The language of the MCQ task. - adapter (Callable[[dict], MCQInput] | MCQDictAdapter): A function or dictionary to adapt the input data to the required MCQInput format. - Must map data from the dataset row to the MCQInput format. - formulation (Formulation, optional): The formulation to use for the task. Defaults to MCFFormulation(). - - Returns: - Callable: A function that generates MCQ prompts based on the given parameters. - """ - adapter_fn = create_adapter_from_dict(adapter) - - def prompt_fn(line, task_name: str): - mcq_input = adapter_fn(line) - if mcq_input is None: - return None - - translation_literals = TRANSLATION_LITERALS[language] - - instruction_val = mcq_input.get("instruction") - instruction = f"{instruction_val}\n" if instruction_val else "" - - context_val = mcq_input.get("context") - context = f"{capitalize(fix_ending_punct(context_val, translation_literals))}\n" if context_val else "" - - question = capitalize(fix_ending_punct(mcq_input["question"], translation_literals)) - answers = [capitalize(fix_ending_punct(str(answer), translation_literals)) for answer in mcq_input["choices"]] - - options = build_choices(answers, formulation, translation_literals) - options = f"{options}\n" if options else "" - answers = build_answers(answers, formulation, translation_literals) - - answer_word = capitalize(translation_literals.answer) - question_word = capitalize(translation_literals.question_word) - - query = MULTI_CHOICE_QA_QUERY.format( - instruction=instruction, - question=question, - context=context, - question_word=question_word, - answer_word=answer_word, - colon=translation_literals.colon, - sentence_space=translation_literals.sentence_space, - options=options, - ) - - return Doc( - task_name=task_name, - query=query, - gold_index=as_list(mcq_input["gold_idx"]), - choices=answers, - instruction=instruction_val, - unconditioned_query=f"{answer_word}{translation_literals.colon}", - ) - - return prompt_fn diff --git a/src/lighteval/tasks/templates/nli.py b/src/lighteval/tasks/templates/nli.py deleted file mode 100644 index 06908401f..000000000 --- a/src/lighteval/tasks/templates/nli.py +++ /dev/null @@ -1,257 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from typing import Callable, Literal - -from typing_extensions import NotRequired, TypedDict - -from lighteval.tasks.requests import Doc -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.adapter_utils import create_adapter_from_dict -from lighteval.tasks.templates.utils.formatting_utils import PUNCT, capitalize, decapitalize, fix_ending_punct -from lighteval.tasks.templates.utils.formulation import CFFormulation, Formulation, HybridFormulation, MCFFormulation -from lighteval.tasks.templates.utils.translation_literals import TRANSLATION_LITERALS, TranslationLiterals -from lighteval.utils.language import Language - - -NLI_TEMPLATE_QUERY_CF = "{instruction}{premise}{word_space}{confirmation_word}{question_mark}" -NLI_TEMPLATE_CONT_CF = "{sentence_space}{label}{comma}{word_space}{hypothesis}" - - -class NLIInput(TypedDict): - """Input for the Natural Language Inference (NLI) task. - - Args: - premise: The premise of the NLI task (e.g. He can speak French) - hypothesis: The hypothesis of the NLI task (e.g. He can speak anglo-saxon language) - gold_idx: The index of the correct relation - instruction (optional): The instruction of the task (e.g. Choose the correct relation between the premise and hypothesis) - """ - - premise: str - hypothesis: str - gold_idx: int - instruction: NotRequired[str] - - -class NLIAdapter(TypedDict): - """Adapter for mapping from the dataset row into the NLIInput format. - - Args: - premise: Column name in the row that contains the premise of the NLI task (e.g. He can speak French) - hypothesis: Column name in the row that contains the hypothesis of the NLI task (e.g. He can speak anglo-saxon language) - gold_idx: Column name in the row that contains the index of the correct relation - instruction (optional): Column name in the row that contains the instruction of the task (e.g. Choose the correct relation between the premise and hypothesis) - """ - - premise: str - hypothesis: str - gold_idx: str - instruction: NotRequired[str] - - -RelationType = Literal["entailment", "neutral", "contradiction"] - - -def _nli_prompt_function_natural( - language: Language, - adapter: Callable[[dict], NLIInput | None] | NLIAdapter, - relations: list[RelationType], -): - """Create a templated prompt function for a Natural Language Inference (NLI) task. - Example tasks: - - ANLI - - XNLI - - Format: - Premise right? | (Yes,No,Also), hypothesis - - Args: - language (Language): The language of the NLI task. - adapter (Callable[[dict], NLIInput] | NLIAdapter): Either a function that takes a dataset row and returns a NLIInput, or a dictionary with keys corresponding to the field names in the dataset row. - Note: Both NLIAdapter and NLIInput are TypeDicts, this means that the caller provides dictionary and doesn't initialize any class! - Note: The gold_idx must be an index in the relations list, indicating the correct relation. The order of relations is important! - - relations (list[Literal["entailment", "neutral", "contradiction"]]): The list of possible relations for the NLI task. - - Returns: - Callable: A function that generates NLI prompts based on the given parameters. - """ - - def get_relation_label(label: RelationType, translation_literals: TranslationLiterals): - if label == "entailment": - return translation_literals.yes - elif label == "contradiction": - return translation_literals.no - elif label == "neutral": - return translation_literals.also - - translation_literals = TRANSLATION_LITERALS[language] - adapter_fn = create_adapter_from_dict(adapter) - - def nli_natural_prompt(line: dict, task_name: str): - labels = [capitalize(get_relation_label(label, translation_literals)) for label in relations] - input_data = adapter_fn(line) - if input_data is None: - return None - - premise, hypothesis, label = input_data["premise"], input_data["hypothesis"], input_data["gold_idx"] - - premise = capitalize(input_data["premise"].rstrip(PUNCT)) - hypothesis = decapitalize(input_data["hypothesis"].rstrip(PUNCT)) - - query = NLI_TEMPLATE_QUERY_CF.format( - instruction=input_data.get("instruction", ""), - premise=premise, - full_stop=translation_literals.full_stop, - confirmation_word=translation_literals.confirmation_word, - question_mark=translation_literals.question_mark, - word_space=translation_literals.word_space, - ) - - choices = [ - NLI_TEMPLATE_CONT_CF.format( - hypothesis=hypothesis, - label=label, - comma=translation_literals.comma, - word_space=translation_literals.word_space, - sentence_space=translation_literals.sentence_space, - ) - for label in labels - ] - - unconditioned_query = NLI_TEMPLATE_QUERY_CF.format( - instruction="", - premise="", - confirmation_word=translation_literals.confirmation_word, - question_mark=translation_literals.question_mark, - word_space="", - full_stop=translation_literals.full_stop, - ) - - return Doc( - task_name=task_name, - query=query, - choices=choices, - gold_index=label, - unconditioned_query=unconditioned_query, - ) - - return nli_natural_prompt - - -def get_nli_prompt_function( - language: Language, - adapter: Callable[[dict], NLIInput | None] | NLIAdapter, - relations: list[RelationType], - formulation: Formulation = MCFFormulation(), -): - """Create a templated prompt function for a Natural Language Inference (NLI) task. - Example tasks: - - ANLI - - XNLI - - Format: - *CF* - Premise, right? | Yes/No/False Hypothesis - - *Hybrid* - Premise - Question: Hypothesis True,False or Neither? - | True/False/Neither - - *MCF* - Premise - Question: Hypothesis? - A. True - B. False - C. Neither - | A/B/C - - Args: - language (Language): The language of the NLI task. - adapter (Callable[[dict], NLIInput] | NLIAdapter): Either a function that takes a dataset row and returns a NLIInput, or a dictionary with keys corresponding to the field names in the dataset row. - Note: Both NLIAdapter and NLIInput are TypeDicts, this means that the caller provides dictionary and doesn't initialize any class! - Note: The gold_idx must be an index in the relations list, indicating the correct relation. The order of relations is important! - relations (list[RelationType]): List of relation types to use for the task (e.g., ["entailment", "contradiction", "neutral"]). - formulation (Formulation, optional): The formulation to use for the task. Defaults to MCFFormulation(). - - Returns: - Callable: A function that generates NLI prompts based on the given parameters. - """ - # We use natural implementation for CF formulation to comply with standard evaluation formats - if isinstance(formulation, CFFormulation): - return _nli_prompt_function_natural(language, adapter, relations) - - translation_literals = TRANSLATION_LITERALS[language] - - def get_relation_label(label: RelationType, translation_literals: TranslationLiterals): - if label == "entailment": - return translation_literals.true - elif label == "contradiction": - return translation_literals.false - elif label == "neutral": - return translation_literals.neither - - adapter_fn = create_adapter_from_dict(adapter) - - # For hybrid we use inlined choices so we use the cf formulation in multichoice prompt fn - mcq_prompt_fn = get_mcq_prompt_function( - language, - {"context": "premise", "question": "hypothesis", "choices": "choices", "gold_idx": "gold_idx"}, - CFFormulation() if isinstance(formulation, HybridFormulation) else formulation, - ) - - def prompt_fn(line: dict, task_name: str): - input_data = adapter_fn(line) - if input_data is None: - return None - - # Template based on discussion here: https://github.com/EleutherAI/lm-evaluation-harness/issues/450 - labels = [capitalize(get_relation_label(label, translation_literals)) for label in relations] - - premise, hypothesis, gold_idx = input_data["premise"], input_data["hypothesis"], input_data["gold_idx"] - premise = fix_ending_punct(capitalize(input_data["premise"]), translation_literals) - hypothesis = input_data["hypothesis"] - if isinstance(formulation, HybridFormulation): - # If we have the neither option move it to the end to be consistent with standard NLI evaluation - rearranged_labels = labels - if "neutral" in relations: - neutral_idx = relations.index("neutral") - rearranged_labels = labels[:neutral_idx] + labels[neutral_idx + 1 :] + [labels[neutral_idx]] - - choices_str = f"{translation_literals.comma}{translation_literals.word_space}".join(rearranged_labels[:-1]) - hypothesis = f"{hypothesis.rstrip(PUNCT)}{translation_literals.sentence_space}{choices_str}{translation_literals.word_space}{translation_literals.or_word}{translation_literals.word_space}{rearranged_labels[-1]}{translation_literals.question_mark}" - - # (hynky1999): Ideally we would not compute logprobs of the Yes/No/Also in CF formulation. However as of right now lighteval doesn't allow to - # use multi-context. - row = { - "instruction": input_data.get("instruction", ""), - "premise": premise, - "hypothesis": hypothesis, - "gold_idx": gold_idx, - "choices": labels, - } - - return mcq_prompt_fn(row, task_name) - - return prompt_fn diff --git a/src/lighteval/tasks/templates/qa.py b/src/lighteval/tasks/templates/qa.py deleted file mode 100644 index 3c5074b7d..000000000 --- a/src/lighteval/tasks/templates/qa.py +++ /dev/null @@ -1,80 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from typing import Callable - -from typing_extensions import NotRequired, TypedDict - -from lighteval.tasks.templates.multichoice import MCQInput, create_adapter_from_dict, get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import CFFormulation -from lighteval.utils.language import Language - - -class QAInput(TypedDict): - question: str - choices: list[str] - context: NotRequired[str] - instruction: NotRequired[str] - - -class QAAdapter(TypedDict): - question: str - context: str - context: NotRequired[str] - instruction: NotRequired[str] - - -def get_qa_prompt_function(language: Language, adapter: Callable[[dict], QAInput | None] | QAAdapter): - """Create a templated prompt function for a QA task. - Example tasks: - - XQuAD - - SQuAD - - Format: - Question: xxx - Answer: | Answer - - Args: - language (Language): The language of the QA task. - adapter (Callable[[dict], QAInput] | QAAdapter): A function or dictionary to adapt the input data to the required QAInput format. - Must map data from the dataset row to the QAInput format. - - Returns: - Callable: A function that generates QA prompts based on the given parameters. - """ - adapter_fn = create_adapter_from_dict(adapter) - - def adapter_for_mcq(line: dict) -> MCQInput | None: - input_data = adapter_fn(line) - if input_data is None: - return None - - choices = list(set(input_data["choices"])) - - return { - **input_data, - "gold_idx": list(range(len(choices))), - "choices": choices, - } - - multichoice_prompt_fn = get_mcq_prompt_function(language, adapter=adapter_for_mcq, formulation=CFFormulation()) - return multichoice_prompt_fn diff --git a/src/lighteval/tasks/templates/translation.py b/src/lighteval/tasks/templates/translation.py deleted file mode 100644 index 6b4c54a62..000000000 --- a/src/lighteval/tasks/templates/translation.py +++ /dev/null @@ -1,158 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from typing import Callable - -from langcodes import standardize_tag -from typing_extensions import NotRequired, TypedDict - -from lighteval.tasks.templates.continuation import get_continuation_prompt_function -from lighteval.tasks.templates.multichoice import create_adapter_from_dict -from lighteval.tasks.templates.utils.formatting_utils import capitalize, fix_ending_punct -from lighteval.tasks.templates.utils.formulation import Formulation, MCFFormulation -from lighteval.tasks.templates.utils.translation_literals import TRANSLATION_LITERALS -from lighteval.utils.language import Language -from lighteval.utils.utils import as_list - - -# Template chosen so that it's not very language-dependent, as it's not clear whether one should use the target or source language. -# It's also the best template based on https://arxiv.org/pdf/2301.07069. - - -TRANSLATION_CONTEXT = "{source_label}{colon}{sentence_space}{source_text}{sentence_space}{target_label}{colon}" - - -# Defined for type hinting only -class TranslationInput(TypedDict): - """Input for the Translation task. - - Args: - source_text: The source text to be translated - target_text: The target text to be translated - instruction (optional): The instruction of the Translation task (e.g. Translate the following text to Turkish) - """ - - source_text: str - target_text: str | list[str] - gold_idx: NotRequired[int | list[int]] - instruction: NotRequired[str] - - -class TranslationAdapter(TypedDict): - """Adapter for mapping from the dataset row into the TranslationInput format. - - Args: - source_text: Column name in the row that contains the source text to be translated - target_text: Column name in the row that contains the target text to be translated - instruction (optional): Column name in the row that contains the instruction of the task (e.g. Translate the following text to Turkish) - """ - - source_text: str - target_text: str - gold_idx: NotRequired[int | list[int]] - instruction: NotRequired[str] - - -def get_translation_prompt_function( - source_language: Language, - target_language: Language, - adapter: Callable[[dict], TranslationInput | None] | TranslationAdapter, - formulation: Formulation = MCFFormulation(), -): - """Create a templated prompt function for a Translation task. - Example tasks: - - WMT2016 - - WMT2017 - - Format: - *CF* - EN: How are you? TR: | Nasılsın? - - *Hybrid* - EN: How are you? TR: - A. Nasılsın? - B. Jak se máš? - Answer: | Nasılsın?/Jak se máš? - - *MCF* - EN: How are you? TR: - A. Nasılsın? - B. Jak se máš? - Answer: | A/B - - Args: - source_language (Language): The source language for translation. - target_language (Language): The target language for translation. - adapter (Callable[[dict], TranslationInput] | TranslationAdapter): Either a function that takes a dataset row and returns a TranslationInput, or a dictionary with keys corresponding to the field names in the dataset row. - Note: Both TranslationAdapter and TranslationInput are TypeDicts, this means that the caller provides dictionary and doesn't initialize any class! - formulation (Formulation, optional): The formulation to use for the task. Defaults to MCFFormulation(). - - Returns: - Callable: A function that generates Translation prompts based on the given parameters. - """ - adapter_fn = create_adapter_from_dict(adapter) - continuation_prompt_fn = get_continuation_prompt_function( - Language.ENGLISH, - {"context": "context", "continuations": "continuations", "gold_idx": "gold_idx"}, - formulation, - fix_formatting=False, - ) - source_translation_literals = TRANSLATION_LITERALS[source_language] - target_translation_literals = TRANSLATION_LITERALS[target_language] - - source_label_string = standardize_tag(source_language.value).upper() - target_label_string = standardize_tag(target_language.value).upper() - - def translation_prompt( - line: dict, - task_name: str, - ): - input_data = adapter_fn(line) - if input_data is None: - return None - - source_text = capitalize(fix_ending_punct(input_data["source_text"], source_translation_literals)) - - context = TRANSLATION_CONTEXT.format( - source_label=source_label_string, - source_text=source_text, - target_label=target_label_string, - colon=":", - sentence_space=" ", - ) - - continuations = [ - capitalize(fix_ending_punct(text, target_translation_literals)) - for text in as_list(input_data["target_text"]) - ] - - return continuation_prompt_fn( - { - "instruction": input_data.get("instruction", ""), - "context": context, - "continuations": continuations, - "gold_idx": input_data.get("gold_idx", list(range(len(continuations)))), - }, - task_name, - ) - - return translation_prompt diff --git a/src/lighteval/tasks/templates/utils/__init__.py b/src/lighteval/tasks/templates/utils/__init__.py deleted file mode 100644 index a732db8d0..000000000 --- a/src/lighteval/tasks/templates/utils/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. diff --git a/src/lighteval/tasks/templates/utils/adapter_utils.py b/src/lighteval/tasks/templates/utils/adapter_utils.py deleted file mode 100644 index f4dcbe80c..000000000 --- a/src/lighteval/tasks/templates/utils/adapter_utils.py +++ /dev/null @@ -1,47 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -from typing import Any, Callable, Mapping, TypeVar - - -AdapterReturnTypeVar = TypeVar("AdapterReturnTypeVar") - - -def create_adapter_from_dict( - adapter: Mapping[str, Any] | Callable[[dict], AdapterReturnTypeVar], -) -> Callable[[dict], AdapterReturnTypeVar]: - """Creates adapter function for the template input from a dict. - - Args: - adapter: Dict of the form {key: value} where value is key in the input dict to get. - - Returns: - Callable[[dict], AdapterReturnTypeVar]: A function that adapts dictionary input to the expected format - """ - if not isinstance(adapter, Mapping): - return adapter - - def adapter_fn(line: dict): - return {key: line[value] for key, value in adapter.items()} - - return adapter_fn # type: ignore diff --git a/src/lighteval/tasks/templates/utils/formatting_utils.py b/src/lighteval/tasks/templates/utils/formatting_utils.py deleted file mode 100644 index 2d2daace0..000000000 --- a/src/lighteval/tasks/templates/utils/formatting_utils.py +++ /dev/null @@ -1,107 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from lighteval.tasks.templates.utils.translation_literals import TranslationLiterals - - -# Contains punctuation covering most of the languages big chunk took from https://stackoverflow.com/questions/9506869/are-there-character-collections-for-all-international-full-stop-punctuations -PUNCT = "᪩？⁈𑩂．꩞𑅃﹗𑂾\u1b7d፧𑅂꡶꘎⁉࠾᪨𑊩𑱂᱿𖩮᥅\U00011f43\U00011f44﹒𑈹𑈸።܂؞꛳\U00010f88𑗍𐩖𑙂\u061d꩟᠉\u1b7e𑗗᰼𑻸؟𑪜꧉𑗉𐽙𖫵𖬷܀꓿᜵𑗏𑁇𑗓𑥄៖𑥆𑗑𑗒꯫'۔𐩗\U00010f86꡷\u2e54｡៕߹⸮.𑇅࠹𛲟꫰ល꤯𐽗᭞𑜼፨𑃁꣏𑇟𖬸𑪛𑜾࠷𝪈?𑃀𑗃！։꣎॥𑗖᭛᠃!၊𖺘⁇𑗌𑑋𖭄᭟\"𑅁𑙁⸼꩝𑗋。꧈꫱𑜽𐽖𑂿᙮។꛷\U00010f89៚᥄𑗕𑗎᪪᭚࠽𑇞𑗊𐽘\u2e53𑗔𖩯𑇍𑻷𐽕𑩃।𑗂𑇆𑁈။᱾𑱁꘏܁᜶‼𑈻‽᪫﹖𑑌𑈼\U00010f87𑗐៙᰻" - - -def decapitalize(word: str): - """Decapitalize the first letter of the string - - Returns: - str: The string with the first letter decapitalized - """ - if len(word) == 0: - return word - return word[0].lower() + word[1:] - - -def capitalize(word: str): - """Capitalize the first letter of the string - - Returns: - str: The string with the first letter capitalized - """ - if len(word) == 0: - return word - return word[0].upper() + word[1:] - - -def fix_ending_punct(ctx: str, translation_literals: TranslationLiterals): - """Fixes the ending punctuation so that it uses the correct punctuation for the language. - E.g in Chinese the "?" is written as "？" - - Returns: - str: The string with corrected punctuation for the language - """ - ctx = ctx.rstrip() - if len(ctx) == 0: - return ctx - if ctx.endswith("?"): - ctx = ctx[:-1] + translation_literals.question_mark - elif ctx.endswith("."): - ctx = ctx[:-1] + translation_literals.full_stop - elif ctx.endswith(","): - ctx = ctx[:-1] + translation_literals.comma - elif ctx.endswith(":"): - ctx = ctx[:-1] + translation_literals.colon - return ctx - - -def punctuation_ends_sentence(text: str, translation_literals: TranslationLiterals): - """Check if the string ends with a sentence-ending punctuation mark. - That's .?!: - It's not perfect method as some languages don't have full stops etc.. - - Returns: - bool: True if the string ends with sentence-ending punctuation, False otherwise - """ - return text.rstrip().endswith( - ( - translation_literals.question_mark, - translation_literals.full_stop, - translation_literals.exclamation_mark, - translation_literals.colon, - translation_literals.semicolon, - ) - ) - - -def fix_capitalization(prefix: str, text: str, translation_literals: TranslationLiterals): - """Fixes the capitalization of the text based on the prefix. - It's based on simple heuristics: - - If the prefix ends with a sentence-ending punctuation mark, the text should be capitalized. - - If the prefix ends with a newline, the text should be capitalized. - - Returns: - str: The text with appropriate capitalization based on the prefix - """ - if len(prefix) == 0: - return capitalize(text) - - if prefix.endswith("\n"): - return capitalize(text) - - return capitalize(text) if punctuation_ends_sentence(prefix, translation_literals) else decapitalize(text) diff --git a/src/lighteval/tasks/templates/utils/formulation.py b/src/lighteval/tasks/templates/utils/formulation.py deleted file mode 100644 index 29cece2fd..000000000 --- a/src/lighteval/tasks/templates/utils/formulation.py +++ /dev/null @@ -1,154 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from dataclasses import dataclass -from string import ascii_uppercase -from typing import Literal - -from lighteval.tasks.templates.utils.translation_literals import TranslationLiterals - - -ChoicePrefix = Literal["Letters", "NativeLetters", "Numbers"] - - -@dataclass -class MCFFormulation: - """MCF Formulation - Presenting the choices as A. B. C. - The target is A, B, C - - Args: - choice_prefix (ChoicePrefix, optional): The choice prefix to use for the task. Defaults to "Letters". - """ - - choice_prefix: ChoicePrefix = "Letters" - name: str = "MCF" - - -@dataclass -class HybridFormulation: - """Hybrid Formulation - Presenting the choices as A. B. C. - The target is then the answer itself not A, B, C - - Args: - choice_prefix (ChoicePrefix, optional): The choice prefix to use for the task. Defaults to "Letters". - """ - - choice_prefix: ChoicePrefix = "Letters" - name: str = "Hybrid" - - -@dataclass -class CFFormulation: - """CF Formulation - No choices are presented, the target is the answer itself - """ - - name: str = "CF" - - -Formulation = CFFormulation | HybridFormulation | MCFFormulation - - -def get_prefix(choice_prefix: ChoicePrefix, translation_literals: TranslationLiterals): - if choice_prefix == "Letters": - return ascii_uppercase - elif choice_prefix == "NativeLetters": - return translation_literals.indices - elif choice_prefix == "Numbers": - return list(map(str, list(range(1, 27)))) - - -def build_choices( - choices: list[str], - formulation: Formulation, - translation_literals: TranslationLiterals, - use_sentence_space: bool = True, -): - """Builds a string version of the choices based on passed formulation for available options presentation. - For Hybrid/MCF, the choices are presented as A. OptionA B. OptionB C. OptionC etc. - For CF no choices are presented - - Args: - choices (list[str]): List of choices to be presented. - formulation (Formulation): The formulation to use for the task. - translation_literals (TranslationLiterals): The translation literals scoped to required language. - use_sentence_space (bool, optional): Whether to use sentence or word space in front of the choice. - The same value should be passed to `build_answers` function to ensure consistent tokenization. - - Defaults to True. - - Returns: - str | None: Formatted choices string for Hybrid/MCF formulations, None for CF formulation - """ - if isinstance(formulation, CFFormulation): - return None - - prefixes = get_prefix(formulation.choice_prefix, translation_literals) - - # Note the "answer_space" being variant before the actual answer key, this ensures consistent tokenization with answers - answer_space = translation_literals.sentence_space if use_sentence_space else translation_literals.word_space - if isinstance(formulation, MCFFormulation): - options = "\n".join( - [ - f"{answer_space}{prefixes[i]}{translation_literals.full_stop}{translation_literals.sentence_space}{c}" - for i, c in enumerate(choices) - ] - ) - else: - options = "\n".join( - [ - f"{translation_literals.sentence_space}{prefixes[i]}{translation_literals.full_stop}{answer_space}{c}" - for i, c in enumerate(choices) - ] - ) - return options - - -def build_answers( - answers: list[str], - formulation: Formulation, - translation_literals: TranslationLiterals, - use_sentence_space: bool = True, -) -> list[str]: - """Builds a string version of the answers based on passed formulation. - For MCF, the answers are presented as A, B, C etc. - For Hybrid/CF, the answers are presented as the answer itself. - - Args: - answers (list[str]): List of answers to be presented. - formulation (Formulation): The formulation to use for the task. - translation_literals (TranslationLiterals): The translation literals scoped to required language. - use_sentence_space (bool, optional): Whether to use sentence or word space in front of the answer. Defaults to True. - The same value should be passed to `build_choices` function to ensure consistent tokenization. - - Returns: - list[str]: List of formatted answer strings - """ - if isinstance(formulation, MCFFormulation): - prefixes = get_prefix(formulation.choice_prefix, translation_literals) - answers = [prefixes[i] for i in range(len(answers))] - - # Same tokenization as with answer key in options! - answer_space = translation_literals.sentence_space if use_sentence_space else translation_literals.word_space - return [f"{answer_space}{a}" for a in answers] diff --git a/src/lighteval/tasks/templates/utils/translation_literals.py b/src/lighteval/tasks/templates/utils/translation_literals.py deleted file mode 100644 index 96cd8fc2c..000000000 --- a/src/lighteval/tasks/templates/utils/translation_literals.py +++ /dev/null @@ -1,1428 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from dataclasses import dataclass, field -from string import ascii_uppercase - -from lighteval.utils.language import Language - - -# TODO(hynky1999): The typing still is not great, it should be able to infer that you can't access the -# attributes that are not defined in the class. Don't want to waste time on this though. -@dataclass -class TranslationLiterals: - # This is just to create nice error messages - language: Language - - question_word: str = None # type: ignore - answer: str = None # type: ignore - confirmation_word: str = None # type: ignore - yes: str = None # type: ignore - no: str = None # type: ignore - also: str = None # type: ignore - cause_word: str = None # type: ignore - effect_word: str = None # type: ignore - or_word: str = None # type: ignore - and_word: str = None # type: ignore - - # NLI - true: str = None # type: ignore - false: str = None # type: ignore - neither: str = None # type: ignore - - # Punctuation - full_stop: str = "." - comma: str = "," - question_mark: str = "?" - exclamation_mark: str = "!" - word_space: str = " " - sentence_space: str = " " - colon: str = ":" - semicolon: str = ";" - - # Indices - indices: list[str] = field(default_factory=lambda: ascii_uppercase) - - def __getattribute__(self, name: str) -> str: - value = super().__getattribute__(name) - if value is None: - raise AttributeError( - f""" -Translation for '{name}' is needed for {self.language}. Please provide its implementation by editing -the 'src/lighteval/tasks/templates/utils/translation_literals.py' -""" - ) - return value - - -TRANSLATION_LITERALS: dict[Language, TranslationLiterals] = { - Language.ACEHNESE: TranslationLiterals(language=Language.ACEHNESE), - Language.AFRIKAANS: TranslationLiterals(language=Language.AFRIKAANS), - Language.AKAN: TranslationLiterals(language=Language.AKAN), - Language.ALBANIAN: TranslationLiterals(language=Language.ALBANIAN), - Language.AMHARIC: TranslationLiterals(language=Language.AMHARIC), - Language.ARABIC: TranslationLiterals( - language=Language.ARABIC, - question_word="سؤال", - answer="إجابة", - confirmation_word="صحيح", - yes="نعم", - no="لا", - also="كذلك", - cause_word="لأن", - effect_word="لذلك", - true="صحيح", - false="خاطئ", - neither="لا هذا ولا ذاك", - or_word="أو", - and_word="و", - full_stop=".", - comma="،", - question_mark="؟", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - indices=["أ", "ب", "ج", "د", "هـ", "و", "ز", "ح"], - ), - Language.ARMENIAN: TranslationLiterals(language=Language.ARMENIAN), - Language.ASSAMESE: TranslationLiterals(language=Language.ASSAMESE), - Language.ASTURIAN: TranslationLiterals(language=Language.ASTURIAN), - Language.AWADHI: TranslationLiterals(language=Language.AWADHI), - Language.AYACUCHO_QUECHUA: TranslationLiterals(language=Language.AYACUCHO_QUECHUA), - Language.AZERBAIJANI: TranslationLiterals(language=Language.AZERBAIJANI), - Language.BALINESE: TranslationLiterals(language=Language.BALINESE), - Language.BAMBARA: TranslationLiterals(language=Language.BAMBARA), - Language.BANJAR: TranslationLiterals(language=Language.BANJAR), - Language.BASHKIR: TranslationLiterals( - language=Language.BASHKIR, - question_word="һорау", - answer="яуап", - confirmation_word="шулаймы", - yes="эйе", - no="яҡ", - also="шулай уҡ", - cause_word="сөнки", - effect_word="шуға", - or_word="йәки", - true="дөрөҫ", - false="ялған", - neither="башҡа", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - indices=["А", "Б", "В", "Г", "Д", "Е"], - ), - Language.BASQUE: TranslationLiterals( - language=Language.BASQUE, - question_word="galdera", - answer="erantzuna", - confirmation_word="ezta", - yes="bai", - no="ez", - also="halaber", - cause_word="izan ere", - effect_word="beraz", - or_word="ala", - true="egia", - false="faltsua", - neither="bat ere ez", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.BELARUSIAN: TranslationLiterals( - language=Language.BELARUSIAN, - question_word="пытанне", - answer="адказ", - confirmation_word="ці не так", - yes="так", - no="не", - also="апроч таго", - cause_word="бо", - effect_word="таму", - true="праўда", - false="няпраўда", - neither="ні тое, ні тое", - or_word="ці", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - indices=["А", "Б", "В", "Г", "Д", "Е"], - ), - Language.BEMBA: TranslationLiterals(language=Language.BEMBA), - Language.BENGALI: TranslationLiterals( - language=Language.BENGALI, - question_word="প্রশ্ন", - answer="উত্তর", - confirmation_word="তাই না", - yes="হ্যাঁ", - no="না", - also="সাথে", - cause_word="কারণ", - effect_word="অতএব", - true="সত্য", - false="মিথ্যা", - neither="ন তাই, ন তাই না", - full_stop="।", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - indices=["ক", "খ", "গ", "ঘ", "ঙ", "চ"], - or_word="বা", - and_word="এবং", - ), - Language.BHOJPURI: TranslationLiterals(language=Language.BHOJPURI), - Language.BIHARI: TranslationLiterals(language=Language.BIHARI), # Deprecated - Language.BOSNIAN: TranslationLiterals(language=Language.BOSNIAN), - Language.BRETON: TranslationLiterals(language=Language.BRETON), - Language.BUGINESE: TranslationLiterals(language=Language.BUGINESE), - Language.BULGARIAN: TranslationLiterals( - language=Language.BULGARIAN, - question_word="въпрос", - answer="отговор", - confirmation_word="вярно", - yes="да", - no="не", - also="също", - cause_word="защото", - effect_word="следователно", - true="истина", - false="лъжа", - neither="нито едното нито другото", - or_word="или", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - indices=["А", "Б", "В", "Г", "Д", "Е"], - ), - Language.BURMESE: TranslationLiterals(language=Language.BURMESE), - Language.CATALAN: TranslationLiterals( - language=Language.CATALAN, - question_word="pregunta", - answer="resposta", - confirmation_word="cert", - yes="sí", - no="no", - also="també", - cause_word="perquè", - effect_word="per tant", - or_word="o", - true="veritable", - false="fals", - neither="cap", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.CEBUANO: TranslationLiterals( - language=Language.CEBUANO, - question_word="pangutana", - answer="tubag", - and_word="ug", - confirmation_word="husto", - yes="oo", - no="dili", - also="ug unya", - cause_word="tungod kay", - effect_word="busa", - or_word="o", - true="tinuod", - false="dili tinuod", - neither="wala", - ), - Language.CENTRAL_ATLAS_TAMAZIGHT: TranslationLiterals(language=Language.CENTRAL_ATLAS_TAMAZIGHT), - Language.CENTRAL_AYMARA: TranslationLiterals(language=Language.CENTRAL_AYMARA), - Language.CENTRAL_KANURI: TranslationLiterals(language=Language.CENTRAL_KANURI), - Language.CENTRAL_KURDISH: TranslationLiterals(language=Language.CENTRAL_KURDISH), - Language.CHHATTISGARHI: TranslationLiterals(language=Language.CHHATTISGARHI), - Language.CHINESE: TranslationLiterals( - language=Language.CHINESE, - question_word="问题", - answer="答案", - confirmation_word="对吗", - yes="是的", - no="不是", - also="而且", - cause_word="因为", - effect_word="所以", - true="真", - false="假", - neither="都不是", - or_word="或", - and_word="和", - full_stop="。", - comma="，", - question_mark="？", - exclamation_mark="！", - word_space="", - sentence_space="", - colon="：", - indices=["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"], - ), - Language.CHOKWE: TranslationLiterals(language=Language.CHOKWE), - Language.CRIMEAN_TATAR: TranslationLiterals(language=Language.CRIMEAN_TATAR), - Language.CROATIAN: TranslationLiterals( - language=Language.CROATIAN, - question_word="pitanje", - answer="odgovor", - confirmation_word="zar ne", - yes="da", - no="ne", - also="također", - cause_word="jer", - effect_word="dakle", - or_word="ili", - true="točno", - false="netočno", - neither="ništa od navedenog", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.CZECH: TranslationLiterals( - language=Language.CZECH, - question_word="otázka", - answer="odpověď", - confirmation_word="že ano", - yes="ano", - no="ne", - also="navíc", - cause_word="protože", - effect_word="a tedy", - or_word="nebo", - true="pravda", - false="nepravda", - neither="ani jedno", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.DANISH: TranslationLiterals( - language=Language.DANISH, - question_word="spørgsmål", - answer="svar", - confirmation_word="ikke", - yes="ja", - no="nej", - also="også", - cause_word="fordi", - effect_word="derfor", - or_word="eller", - true="sandt", - false="falsk", - neither="ingen af delene", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.DARI: TranslationLiterals(language=Language.DARI), - Language.DIVEHI: TranslationLiterals(language=Language.DIVEHI), - Language.DUTCH: TranslationLiterals( - language=Language.DUTCH, - question_word="vraag", - answer="antwoord", - confirmation_word="toch", - yes="ja", - no="nee", - also="ook", - cause_word="want", - effect_word="dus", - or_word="of", - true="waar", - false="onwaar", - neither="geen van beide", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.DYULA: TranslationLiterals(language=Language.DYULA), - Language.DZONGKHA: TranslationLiterals(language=Language.DZONGKHA), - Language.EASTERN_PANJABI: TranslationLiterals(language=Language.EASTERN_PANJABI), - Language.EASTERN_YIDDISH: TranslationLiterals(language=Language.EASTERN_YIDDISH), - Language.EGYPTIAN_ARABIC: TranslationLiterals(language=Language.EGYPTIAN_ARABIC), - Language.ENGLISH: TranslationLiterals( - language=Language.ENGLISH, - question_word="question", - answer="answer", - confirmation_word="right", - yes="yes", - no="no", - also="also", - cause_word="because", - effect_word="therefore", - true="true", - false="false", - neither="neither", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - or_word="or", - and_word="and", - ), - Language.ESPERANTO: TranslationLiterals(language=Language.ESPERANTO), - Language.ESTONIAN: TranslationLiterals( - language=Language.ESTONIAN, - question_word="küsimus", - answer="vastus", - confirmation_word="eks", - yes="jah", - no="ei", - also="samuti", - cause_word="sest", - effect_word="seega", - or_word="või", - true="õige", - false="vale", - neither="mitte kumbki", - ), - Language.EWE: TranslationLiterals(language=Language.EWE), - Language.FAROESE: TranslationLiterals(language=Language.FAROESE), - Language.FIJIAN: TranslationLiterals(language=Language.FIJIAN), - Language.FINNISH: TranslationLiterals( - language=Language.FINNISH, - question_word="kysymys", - answer="vastaus", - confirmation_word="eikö niin", - yes="kyllä", - no="ei", - also="myös", - cause_word="koska", - effect_word="siksi", - or_word="tai", - true="totta", - false="tarua", - neither="ei kumpikaan", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.FON: TranslationLiterals(language=Language.FON), - Language.FRENCH: TranslationLiterals( - language=Language.FRENCH, - question_word="question", - answer="réponse", - confirmation_word="n'est-ce pas", - yes="oui", - no="non", - also="de plus", - cause_word="parce que", - effect_word="donc", - or_word="ou", - and_word="et", - true="vrai", - false="faux", - neither="aucun des deux", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - ), - Language.FRIULIAN: TranslationLiterals(language=Language.FRIULIAN), - Language.GALICIAN: TranslationLiterals( - language=Language.GALICIAN, - question_word="pregunta", - answer="resposta", - confirmation_word="certo", - yes="si", - no="non", - also="tamén", - cause_word="porque", - effect_word="polo tanto", - or_word="ou", - true="verdadeiro", - false="falso", - neither="ningún", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.GANDA: TranslationLiterals(language=Language.GANDA), - Language.GEORGIAN: TranslationLiterals(language=Language.GEORGIAN), - Language.GERMAN: TranslationLiterals( - language=Language.GERMAN, - question_word="frage", - answer="antwort", - confirmation_word="richtig", - yes="ja", - no="nein", - also="auch", - cause_word="weil", - effect_word="deshalb", - or_word="oder", - true="wahr", - false="falsch", - neither="weder noch", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.GREEK: TranslationLiterals( - language=Language.GREEK, - question_word="ερώτηση", - answer="απάντηση", - confirmation_word="σωστά", - yes="ναι", - no="όχι", - also="επίσης", - cause_word="επειδή", - effect_word="άρα", - or_word="ή", - true="σωστό", - false="λάθος", - neither="καμία απάντηση", - full_stop=".", - comma=",", - question_mark=";", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon="·", - ), - Language.GUARANI: TranslationLiterals(language=Language.GUARANI), - Language.GUJARATI: TranslationLiterals( - language=Language.GUJARATI, - question_word="પ્રશ્ન", - answer="જવાબ", - confirmation_word="ખરું ને", - yes="હા", - no="ના", - also="અને", - cause_word="કારણ કે", - effect_word="તેથી", - true="સાચું", - false="ખોટું", - neither="ન તો આ, ન તે", - or_word="અથવા", - and_word="અને", - full_stop="।", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - indices=["અ", "આ", "ઇ", "ઈ", "ઉ", "ઊ"], - ), - Language.HAITIAN: TranslationLiterals( - # From https://github.com/EleutherAI/lm-evaluation-harness/blob/0845b588303f1f59af98dd1c5bdbd78a9e75a1e2/lm_eval/tasks/xcopa/utils.py - language=Language.HAITIAN, - cause_word="poukisa", - effect_word="donk sa", - ), - Language.HAITIAN_CREOLE: TranslationLiterals(language=Language.HAITIAN_CREOLE), - Language.HALH_MONGOLIAN: TranslationLiterals(language=Language.HALH_MONGOLIAN), - Language.HAUSA: TranslationLiterals(language=Language.HAUSA), - Language.HEBREW: TranslationLiterals(language=Language.HEBREW), - Language.HINDI: TranslationLiterals( - language=Language.HINDI, - question_word="सवाल", - answer="उत्तर", - confirmation_word="है ना", - yes="हाँ", - no="नहीं", - also="साथ ही", - cause_word="क्योंकि", - effect_word="इसलिए", - true="सत्य", - false="असत्य", - neither="न तो यह, न वह", - or_word="या", - and_word="और", - full_stop="।", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - indices=["क", "ख", "ग", "घ", "ङ", "च"], - ), - Language.HUNGARIAN: TranslationLiterals( - language=Language.HUNGARIAN, - question_word="kérdés", - answer="válasz", - confirmation_word="ugye", - yes="igen", - no="nem", - also="is", - cause_word="mert", - effect_word="ezért", - or_word="vagy", - true="igaz", - false="hamis", - neither="egyik sem", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.ICELANDIC: TranslationLiterals( - language=Language.ICELANDIC, - question_word="spurning", - answer="svar", - confirmation_word="ekki satt", - yes="já", - no="nei", - also="líka", - cause_word="af því að", - effect_word="þess vegna", - or_word="eða", - true="satt", - false="ósatt", - neither="hvorugt", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.IGBO: TranslationLiterals(language=Language.IGBO), - Language.ILOCANO: TranslationLiterals(language=Language.ILOCANO), - Language.INDONESIAN: TranslationLiterals( - language=Language.INDONESIAN, - question_word="pertanyaan", - answer="jawaban", - confirmation_word="kan", - yes="ya", - no="tidak", - also="juga", - cause_word="karena", - effect_word="maka", - or_word="atau", - true="benar", - false="salah", - neither="tidak satu pun", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.IRISH: TranslationLiterals(language=Language.IRISH), - Language.ITALIAN: TranslationLiterals( - language=Language.ITALIAN, - question_word="domanda", - answer="risposta", - confirmation_word="vero", - yes="sì", - no="no", - also="inoltre", - cause_word="perchè", - effect_word="quindi", - or_word="o", - true="vero", - false="falso", - neither="nessuno dei due", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.JAPANESE: TranslationLiterals( - language=Language.JAPANESE, - question_word="質問", - answer="回答", - confirmation_word="でしょうか", - yes="はい", - no="いいえ", - also="また", - cause_word="なので", - effect_word="なぜなら", - or_word="または", - true="正解", - false="不正解", - neither="どちらでもない", - full_stop="。", - comma="、", - question_mark="？", - exclamation_mark="！", - word_space="", - sentence_space="", - colon="：", - semicolon="；", - ), - Language.JAVANESE: TranslationLiterals(language=Language.JAVANESE), - Language.JINGPHO: TranslationLiterals(language=Language.JINGPHO), - Language.KABIYE: TranslationLiterals(language=Language.KABIYE), - Language.KABUVERDIANU: TranslationLiterals(language=Language.KABUVERDIANU), - Language.KABYLE: TranslationLiterals(language=Language.KABYLE), - Language.KAMBA: TranslationLiterals(language=Language.KAMBA), - Language.KANNADA: TranslationLiterals(language=Language.KANNADA), - Language.KASHMIRI: TranslationLiterals(language=Language.KASHMIRI), - Language.KAZAKH: TranslationLiterals(language=Language.KAZAKH), - Language.KHMER: TranslationLiterals(language=Language.KHMER), - Language.KIKONGO: TranslationLiterals(language=Language.KIKONGO), - Language.KIKUYU: TranslationLiterals(language=Language.KIKUYU), - Language.KIMBUNDU: TranslationLiterals(language=Language.KIMBUNDU), - Language.KINYARWANDA: TranslationLiterals(language=Language.KINYARWANDA), - Language.KIRGHIZ: TranslationLiterals(language=Language.KIRGHIZ), - Language.KOREAN: TranslationLiterals( - language=Language.KOREAN, - confirmation_word="맞죠", - yes="예", - no="아니오", - ), - Language.KURDISH: TranslationLiterals(language=Language.KURDISH), - Language.KYRGYZ: TranslationLiterals(language=Language.KYRGYZ), - Language.LAO: TranslationLiterals(language=Language.LAO), - Language.LATGALIAN: TranslationLiterals(language=Language.LATGALIAN), - Language.LATIN: TranslationLiterals(language=Language.LATIN), - Language.LATVIAN: TranslationLiterals(language=Language.LATVIAN), - Language.LIGURIAN: TranslationLiterals(language=Language.LIGURIAN), - Language.LIMBURGISH: TranslationLiterals(language=Language.LIMBURGISH), - Language.LINGALA: TranslationLiterals(language=Language.LINGALA), - Language.LITHUANIAN: TranslationLiterals(language=Language.LITHUANIAN), - Language.LOMBARD: TranslationLiterals(language=Language.LOMBARD), - Language.LUBA_KASAI: TranslationLiterals(language=Language.LUBA_KASAI), - Language.LUO: TranslationLiterals(language=Language.LUO), - Language.LUXEMBOURGISH: TranslationLiterals(language=Language.LUXEMBOURGISH), - Language.MACEDONIAN: TranslationLiterals( - language=Language.MACEDONIAN, - question_word="прашање", - answer="одговор", - confirmation_word="зарем не", - yes="да", - no="не", - also="исто така", - cause_word="бидејќи", - effect_word="затоа", - or_word="или", - true="точно", - false="не точно", - neither="ништо од наведеното", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.MAGAHI: TranslationLiterals(language=Language.MAGAHI), - Language.MAITHILI: TranslationLiterals(language=Language.MAITHILI), - Language.MALAGASY: TranslationLiterals(language=Language.MALAGASY), - Language.MALAY: TranslationLiterals(language=Language.MALAY), - Language.MALAYALAM: TranslationLiterals(language=Language.MALAYALAM), - Language.MALTESE: TranslationLiterals(language=Language.MALTESE), - Language.MAORI: TranslationLiterals(language=Language.MAORI), - Language.MARATHI: TranslationLiterals(language=Language.MARATHI), - Language.MEITEI: TranslationLiterals(language=Language.MEITEI), - Language.MESOPOTAMIAN_ARABIC: TranslationLiterals(language=Language.MESOPOTAMIAN_ARABIC), - Language.MINANGKABAU: TranslationLiterals(language=Language.MINANGKABAU), - Language.MIZO: TranslationLiterals(language=Language.MIZO), - Language.MODERN_STANDARD_ARABIC: TranslationLiterals(language=Language.MODERN_STANDARD_ARABIC), - Language.MOROCCAN_ARABIC: TranslationLiterals(language=Language.MOROCCAN_ARABIC), - Language.MOSSI: TranslationLiterals(language=Language.MOSSI), - Language.NAJDI_ARABIC: TranslationLiterals(language=Language.NAJDI_ARABIC), - Language.NEPALI: TranslationLiterals(language=Language.NEPALI), - Language.NIGERIAN_FULFULDE: TranslationLiterals(language=Language.NIGERIAN_FULFULDE), - Language.NORTHERN_KURDISH: TranslationLiterals(language=Language.NORTHERN_KURDISH), - Language.NORTHERN_SOTHO: TranslationLiterals(language=Language.NORTHERN_SOTHO), - Language.NORTHERN_UZBEK: TranslationLiterals(language=Language.NORTHERN_UZBEK), - Language.NORTH_AZERBAIJANI: TranslationLiterals(language=Language.NORTH_AZERBAIJANI), - Language.NORTH_LEVANTINE_ARABIC: TranslationLiterals(language=Language.NORTH_LEVANTINE_ARABIC), - Language.NORWEGIAN: TranslationLiterals( - language=Language.NORWEGIAN, - question_word="spørsmål", - answer="svar", - confirmation_word="ikke sant", - yes="ja", - no="nei", - also="i tillegg", - cause_word="fordi", - effect_word="derfor", - or_word="eller", - true="sant", - false="usant", - neither="ingen av delene", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.NORWEGIAN_BOKMAL: TranslationLiterals(language=Language.NORWEGIAN_BOKMAL), - Language.NORWEGIAN_NYNORSK: TranslationLiterals(language=Language.NORWEGIAN_NYNORSK), - Language.NUER: TranslationLiterals(language=Language.NUER), - Language.NYANJA: TranslationLiterals(language=Language.NYANJA), - Language.OCCITAN: TranslationLiterals(language=Language.OCCITAN), - Language.ODIA: TranslationLiterals(language=Language.ODIA), - Language.ORIYA: TranslationLiterals(language=Language.ORIYA), - Language.PANGASINAN: TranslationLiterals(language=Language.PANGASINAN), - Language.PAPIAMENTO: TranslationLiterals(language=Language.PAPIAMENTO), - Language.PASHTO: TranslationLiterals(language=Language.PASHTO), - Language.PERSIAN: TranslationLiterals(language=Language.PERSIAN), - Language.PLATEAU_MALAGASY: TranslationLiterals(language=Language.PLATEAU_MALAGASY), - Language.POLISH: TranslationLiterals( - language=Language.POLISH, - question_word="pytanie", - answer="odpowiedź", - confirmation_word="prawda", - yes="tak", - no="nie", - also="ponadto", - cause_word="ponieważ", - effect_word="więc", - or_word="lub", - true="prawda", - false="fałsz", - neither="ani jedno ani drugie", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.PORTUGUESE: TranslationLiterals( - language=Language.PORTUGUESE, - question_word="pergunta", - answer="resposta", - confirmation_word="certo", - yes="sim", - no="não", - also="adicionalmente", - cause_word="porque", - effect_word="logo", - or_word="ou", - true="verdadeiro", - false="falso", - neither="nenhum", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.PUNJABI: TranslationLiterals( - language=Language.PUNJABI, - question_word="ਸਵਾਲ", - answer="ਜਵਾਬ", - confirmation_word="ਹਾਂ ਨਾ", - yes="ਹਾਂ", - no="ਨਹੀਂ", - also="ਨਾਲ ਹੀ", - cause_word="ਕਿਉਂਕਿ", - effect_word="ਇਸ ਲਈ", - true="ਸੱਚ", - false="ਝੂਠ", - neither="ਨਾ ਤਾਂ, ਨਾ", - full_stop="।", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - or_word="ਜਾਂ", - and_word="ਅਤੇ", - ), - Language.QUECHUA: TranslationLiterals( - # From https://github.com/EleutherAI/lm-evaluation-harness/blob/0845b588303f1f59af98dd1c5bdbd78a9e75a1e2/lm_eval/tasks/xcopa/utils.py - language=Language.QUECHUA, - cause_word="imataq", - effect_word="chaymi", - ), - Language.ROMANIAN: TranslationLiterals( - language=Language.ROMANIAN, - question_word="întrebare", - answer="răspuns", - confirmation_word="corect", - yes="da", - no="nu", - also="de asemenea", - cause_word="pentru că", - effect_word="prin urmare", - or_word="sau", - and_word="și", - true="adevărat", - false="fals", - neither="nici unul", - ), - Language.RUNDI: TranslationLiterals(language=Language.RUNDI), - Language.RUSSIAN: TranslationLiterals( - language=Language.RUSSIAN, - question_word="вопрос", - answer="ответ", - confirmation_word="верно", - yes="да", - no="нет", - also="к тому же", - cause_word="потому что", - effect_word="поэтому", - true="истина", - false="ложь", - neither="ни то ни другое", - or_word="или", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - indices=["А", "Б", "В", "Г", "Д", "Е"], - ), - Language.SAMOAN: TranslationLiterals(language=Language.SAMOAN), - Language.SANGO: TranslationLiterals(language=Language.SANGO), - Language.SANSKRIT: TranslationLiterals(language=Language.SANSKRIT), - Language.SANTALI: TranslationLiterals(language=Language.SANTALI), - Language.SARDINIAN: TranslationLiterals(language=Language.SARDINIAN), - Language.SCOTTISH_GAELIC: TranslationLiterals(language=Language.SCOTTISH_GAELIC), - # Latin serbian script for future when separating scipts - # Language.SERBIAN_LATIN: TranslationLiterals(language=Language.SERBIAN_LATIN, - # question_word="pitanje", - # answer="odgovor", - # confirmation_word="zar ne", - # yes="da", - # no="ne", - # also="takođe", - # cause_word="jer", - # effect_word="dakle", - # or_word="ili", - # true="tačno", - # false="netačno", - # neither="ništa od navedenog", - # ), - Language.SERBIAN: TranslationLiterals( - language=Language.SERBIAN, - question_word="питање", - answer="одговор", - confirmation_word="зар не", - yes="да", - no="не", - also="такође", - cause_word="јер", - effect_word="дакле", - or_word="или", - true="тачно", - false="нетачно", - neither="ништа од наведеног", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.SERBOCROATIAN: TranslationLiterals(language=Language.SERBOCROATIAN), # Deprecated - Language.SHAN: TranslationLiterals( - language=Language.SHAN, - question_word="ၶေႃႈထၢမ်", - answer="ၶေႃႈတွပ်ႇ", - confirmation_word="ၸွင်ႇၸႂ်ႈ", - yes="ၸႂ်ႈ", - no="ဢမ်ႇ", - also="လႄႈ", - cause_word="ၵွပ်ႈပိူဝ်ႈ", - effect_word="ၵွပ်ႈၼၼ်", - true="တႄႉ", - false="ဢမ်ႇတႄႉ", - neither="ဢမ်ႇၸႂ်ႈတင်းသွင်ဢၼ်", - or_word="ဢမ်ႇၼၼ်", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space="", - sentence_space=" ", - colon=":", - indices=["ၵ", "ၶ", "င", "ၸ", "သ", "ၺ"], - ), - Language.SHONA: TranslationLiterals(language=Language.SHONA), - Language.SICILIAN: TranslationLiterals(language=Language.SICILIAN), - Language.SILESIAN: TranslationLiterals(language=Language.SILESIAN), - Language.SINDHI: TranslationLiterals(language=Language.SINDHI), - Language.SINHALA: TranslationLiterals(language=Language.SINHALA), - Language.SLOVAK: TranslationLiterals( - language=Language.SLOVAK, - question_word="otázka", - answer="odpoveď", - confirmation_word="že áno", - yes="áno", - no="nie", - also="taktiež", - cause_word="pretože", - effect_word="takže", - or_word="alebo", - true="pravda", - false="nepravda", - neither="ani jeden", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.SLOVENIAN: TranslationLiterals(language=Language.SLOVENIAN), - Language.SOMALI: TranslationLiterals(language=Language.SOMALI), - Language.SORANI: TranslationLiterals(language=Language.SORANI), - Language.SOUTHERN_PASHTO: TranslationLiterals(language=Language.SOUTHERN_PASHTO), - Language.SOUTHERN_SOTHO: TranslationLiterals(language=Language.SOUTHERN_SOTHO), - Language.SOUTHWESTERN_DINKA: TranslationLiterals(language=Language.SOUTHWESTERN_DINKA), - Language.SOUTH_AZERBAIJANI: TranslationLiterals(language=Language.SOUTH_AZERBAIJANI), - Language.SOUTH_LEVANTINE_ARABIC: TranslationLiterals(language=Language.SOUTH_LEVANTINE_ARABIC), - Language.SPANISH: TranslationLiterals( - language=Language.SPANISH, - question_word="pregunta", - answer="respuesta", - confirmation_word="cierto", - yes="sí", - no="no", - also="también", - cause_word="porque", - effect_word="por lo tanto", - or_word="o", - true="verdadero", - false="falso", - neither="ninguno", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.STANDARD_LATVIAN: TranslationLiterals(language=Language.STANDARD_LATVIAN), - Language.STANDARD_MALAY: TranslationLiterals(language=Language.STANDARD_MALAY), - Language.STANDARD_TIBETAN: TranslationLiterals(language=Language.STANDARD_TIBETAN), - Language.SUNDANESE: TranslationLiterals(language=Language.SUNDANESE), - Language.SWAHILI: TranslationLiterals( - language=Language.SWAHILI, - question_word="swali", - answer="jibu", - confirmation_word="sahihi", - yes="ndiyo", - no="hapana", - also="pia", - cause_word="kwa sababu", - effect_word="kwa hiyo", - true="kweli", - false="uongo", - neither="hakuna kati ya hizo", - or_word="au", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - ), - Language.SWATI: TranslationLiterals(language=Language.SWATI), - Language.SWEDISH: TranslationLiterals( - language=Language.SWEDISH, - question_word="fråga", - answer="svar", - confirmation_word="eller hur", - yes="ja", - no="nej", - also="också", - cause_word="eftersom", - effect_word="därför att", - or_word="eller", - true="sant", - false="falskt", - neither="ingendera", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.TAGALOG: TranslationLiterals( - language=Language.TAGALOG, - question_word="tanong", - answer="sagot", - and_word="at", - confirmation_word="tama ba", - also="at saka", - yes="oo", - no="hindi", - cause_word="dahil", - effect_word="kaya", - or_word="o", - true="totoo", - false="hindi totoo", - neither="wala", - ), - Language.TAJIK: TranslationLiterals(language=Language.TAJIK), - Language.TAMASHEQ: TranslationLiterals(language=Language.TAMASHEQ), - Language.TAMIL: TranslationLiterals( - language=Language.TAMIL, - question_word="கேள்வி", - answer="பதில்", - confirmation_word="இல்லையா", - yes="ஆம்", - no="இல்லை", - also="மேலும்", - cause_word="ஏனெனில்", - effect_word="எனவே", - true="உண்மை", - false="பொய்", - neither="இல்லை, இல்லை", - full_stop="।", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - or_word="அல்லது", - and_word="மற்றும்", - ), - Language.TATAR: TranslationLiterals( - language=Language.TATAR, - question_word="сорау", - answer="җавап", - confirmation_word="шулай түгелме", - yes="әйе", - no="юк", - also="шулай ук", - cause_word="чөнки", - effect_word="шуңа күрә", - or_word="яки", - true="дөрес", - false="ялган", - neither="бер генә дә", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - indices=["А", "Б", "В", "Г", "Д", "Е"], - ), - Language.TAIZZI_ADENI_ARABIC: TranslationLiterals(language=Language.TAIZZI_ADENI_ARABIC), - Language.TELUGU: TranslationLiterals( - language=Language.TELUGU, - question_word="ప్రశ్న", - answer="జవాబు", - confirmation_word="కదా", - yes="అవును", - no="కాదు", - also="అలాగే", - cause_word="ఎందుకంటే", - effect_word="అందువలన", - or_word="లేదా", - true="నిజం", - false="తప్పు", - neither="ఏదీ కాదు", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - indices=["ఎ", "బి", "సి", "డి", "ఇ"], - ), - Language.THAI: TranslationLiterals( - language=Language.THAI, - question_word="คำถาม", - answer="คำตอบ", - confirmation_word="ใช่ไหม", - yes="ใช่", - no="ไม่", - also="และ", - cause_word="เพราะ", - effect_word="ดังนั้น", - true="จริง", - false="เท็จ", - neither="ไม่ใช่ทั้งสองอย่าง", - or_word="หรือ", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space="", - sentence_space=" ", - colon=":", - indices=["๑", "๒", "๓", "๔", "๕", "๖", "๗", "๘", "๙", "๐"], - ), - Language.TIGRINYA: TranslationLiterals(language=Language.TIGRINYA), - Language.TOK_PISIN: TranslationLiterals(language=Language.TOK_PISIN), - Language.TOSK_ALBANIAN: TranslationLiterals(language=Language.TOSK_ALBANIAN), - Language.TSONGA: TranslationLiterals(language=Language.TSONGA), - Language.TSWANA: TranslationLiterals(language=Language.TSWANA), - Language.TUMBUKA: TranslationLiterals(language=Language.TUMBUKA), - Language.TUNISIAN_ARABIC: TranslationLiterals(language=Language.TUNISIAN_ARABIC), - Language.TURKISH: TranslationLiterals( - language=Language.TURKISH, - question_word="soru", - answer="cevap", - confirmation_word="değil mi", - yes="evet", - no="hayır", - also="ayrıca", - cause_word="çünkü", - effect_word="bu yüzden", - true="doğru", - false="yanlış", - neither="hiçbiri", - or_word="veya", - and_word="ve", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - ), - Language.TURKMEN: TranslationLiterals(language=Language.TURKMEN), - Language.TWI: TranslationLiterals(language=Language.TWI), - Language.UDMURT: TranslationLiterals( - language=Language.UDMURT, - question_word="юан", - answer="валэктон", - confirmation_word="озьы-а", - yes="бен", - no="ӧвӧл", - also="озьы ик", - cause_word="малы ке шуоно", - effect_word="соин ик", - true="шонерлык", - false="пӧяськон", - neither="мукет", - or_word="яке", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - indices=["А", "Б", "В", "Г", "Д", "Е"], - ), - Language.UKRAINIAN: TranslationLiterals( - language=Language.UKRAINIAN, - question_word="питання", - answer="відповідь", - confirmation_word="правильно", - yes="так", - no="ні", - also="також", - cause_word="тому що", - effect_word="отже", - or_word="або", - true="правда", - false="неправда", - neither="ні те, ні інше", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - indices=["А", "Б", "В", "Г", "Д", "Е"], - ), - Language.UMBUNDU: TranslationLiterals(language=Language.UMBUNDU), - Language.URDU: TranslationLiterals( - language=Language.URDU, - question_word="سوال", - answer="جواب", - confirmation_word="نا", - yes="ہاں", - no="نہیں", - also="اور", - cause_word="کیونکہ", - effect_word="اس لئے", - or_word="یا", - true="درست", - false="غلط", - neither="کوئی نہیں", - full_stop="۔", - comma="،", - question_mark="؟", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon="؛", - ), - Language.UYGHUR: TranslationLiterals(language=Language.UYGHUR), - Language.UZBEK: TranslationLiterals( - language=Language.UZBEK, - question_word="savol", - answer="javob", - confirmation_word="to'g'ri", - yes="ha", - no="yo'q", - also="shuningdek", - or_word="yoki", - and_word="va", - cause_word="chunki", - effect_word="shuning uchun", - true="rost", - false="yolg'on", - neither="hech biri", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - ), - Language.VENETIAN: TranslationLiterals(language=Language.VENETIAN), - Language.VIETNAMESE: TranslationLiterals( - language=Language.VIETNAMESE, - question_word="câu hỏi", - answer="trả lời", - confirmation_word="đúng", - yes="có", - no="không", - also="cũng", - cause_word="vì", - effect_word="do đó", - or_word="hoặc", - true="đúng", - false="sai", - neither="không đúng cũng không sai", - full_stop=".", - comma=",", - question_mark="?", - exclamation_mark="!", - word_space=" ", - sentence_space=" ", - colon=":", - semicolon=";", - ), - Language.WAR: TranslationLiterals(language=Language.WAR), - Language.WARAY: TranslationLiterals(language=Language.WARAY), - Language.WELSH: TranslationLiterals(language=Language.WELSH), - Language.WESTERN_FRISIAN: TranslationLiterals(language=Language.WESTERN_FRISIAN), - Language.WESTERN_PERSIAN: TranslationLiterals(language=Language.WESTERN_PERSIAN), - Language.WEST_CENTRAL_OROMO: TranslationLiterals(language=Language.WEST_CENTRAL_OROMO), - Language.WOLOF: TranslationLiterals(language=Language.WOLOF), - Language.XHOSA: TranslationLiterals(language=Language.XHOSA), - Language.YIDDISH: TranslationLiterals(language=Language.YIDDISH), - Language.YORUBA: TranslationLiterals(language=Language.YORUBA), - Language.YUE_CHINESE: TranslationLiterals(language=Language.YUE_CHINESE), - Language.ZULU: TranslationLiterals(language=Language.ZULU), -} diff --git a/src/lighteval/utils/__init__.py b/src/lighteval/utils/__init__.py deleted file mode 100644 index a732db8d0..000000000 --- a/src/lighteval/utils/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. diff --git a/src/lighteval/utils/cache_management.py b/src/lighteval/utils/cache_management.py deleted file mode 100644 index 962f8b083..000000000 --- a/src/lighteval/utils/cache_management.py +++ /dev/null @@ -1,440 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import functools -import hashlib -import json -import logging -import os -from dataclasses import asdict, dataclass -from pathlib import Path -from typing import Callable, List, Set, Tuple, Union - -import pandas as pd -from datasets import Dataset, load_dataset - -from lighteval.models.abstract_model import ModelConfig -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.lighteval_task import LightevalTaskConfig -from lighteval.tasks.registry import Registry -from lighteval.tasks.requests import Doc, SamplingMethod -from lighteval.utils.utils import as_list - - -logger = logging.getLogger(__name__) - - -@dataclass -class TaskID: - """A unique ID for a grouping of task samples. It relies on the task name, - the task config (which gives the task_hash), and the sampling method (linked to - the metric type) - """ - - task_name: str - task_hash: str - sampling_method: SamplingMethod - - def __str__(self): - return f"{self.task_name} ({self.task_hash}, {self.sampling_method.name})" - - def __hash__(self): - return int.from_bytes(hashlib.sha256(str(self).encode()).digest(), byteorder="big") - - -class SampleCache: - """Disk-based cache for sample evaluation results using HuggingFace datasets. - The model hash is a hash of the model config, to make sure we rerun the eval if any parameter changes - (generation param, model version, etc). - - Cache Structure: - - {cache_dir}/ - - {model_name}/ - - {model_hash}/ - - {task_name}/ - - {task_hash}/ dataset dict, where splits are SamplingMethod - """ - - def __init__(self, model_config: ModelConfig): - """Initialize the sample cache. - - Args: - model_config: Configuration for the model being cached - """ - self.model_config = model_config - self.model_hash = self.get_model_hash(model_config) - - self.cache_dir = ( - Path(os.path.expanduser(self.model_config.cache_dir)) / self.model_config.model_name / self.model_hash - ) - self.cache_dir.mkdir(parents=True, exist_ok=True) - - self.registry = None - - self.existing_indices = self._load_cached_indices() - # Caching the task_hashes to avoid grabbing the registry all the time - self._task_hashes = {} - - def _init_registry(self, registry: Registry): - self.registry = registry - - def _load_cached_indices(self) -> dict: - """Loads all indices for samples which are properly cached. We recursively search for all available tasks and files. - - Returns: - dict: Dictionary mapping task names to lists of cached sample indices - """ - logger.info("[CACHING] Initializing data cache") - cached_indices = {} - cache_dir = self.cache_dir - - if not cache_dir.exists(): - return cached_indices - - for cache_file in cache_dir.rglob("*.parquet"): - try: - # cache_file.parts gives all the subfolders of the url, up to the file name - # last 3 are task_name/task_hash/file_name.parquet, so we take -3 and -2 - task_name, task_hash = cache_file.parts[-3:-1] - sampling_method = SamplingMethod[cache_file.stem] # removes the file extension - task_id = TaskID(task_name, task_hash, sampling_method) - - full_dataset = load_dataset("parquet", data_files=str(cache_file), split="train") - sample_ids = [] - for row in full_dataset: - try: - # We only save indices of correctly formatted samples, though this means we need to load each at least once - self._load_sample(row) - cur_sample = row["sample_id"] - sample_ids.append(cur_sample) - except Exception: - continue - - cached_indices[task_id] = sample_ids - logger.info( - f"[CACHING] Loaded {len(sample_ids)} cached indices for task '{str(task_id)} from {cache_file}" - ) - except Exception as e: - logger.warning(f"Error loading cached indices from {cache_file}: {e}") - - return cached_indices - - def get_model_hash(self, model_config: ModelConfig) -> str: - """Create a hash for model configuration. - - Returns: - str: A 16-character hexadecimal hash of the model configuration - """ - # Use Pydantic's model_dump instead of asdict for BaseModel - config_dict = model_config.model_dump() - config_str = json.dumps(config_dict, sort_keys=True, default=str) - return hashlib.sha256(config_str.encode()).hexdigest()[:16] - - def _get_task_hash(self, full_task_name: str) -> str: - """Builds a task_hash from the LightevalTaskConfig loaded from the task name and the registry. - - Args: - full_task_name (str): task name as provided to the registry (supports both task|few_shot and legacy suite|task|few_shot) - - Returns: - str: a hash of the task config in its current state in the registry, or the NO_HASH string if the - registry has not been preloaded - """ - if self.registry is None: - logger.warning( - "The task registry was not provided to the cache config. We can't test if the current task has the same hash as the saved tasks." - ) - return "NO_HASH" - if full_task_name not in self._task_hashes: - parts = full_task_name.split("|") - if len(parts) == 3: - # Legacy: suite|task|few_shot -> ignore suite - _, task_name, _ = parts - elif len(parts) == 2: - task_name, _ = parts - else: - task_name = parts[0] - - task_configs: list[LightevalTaskConfig] = self.registry.task_to_configs[task_name] - # Use deterministic ordering based on string repr - config_strs = sorted([cfg.__str__(lite=True) for cfg in task_configs]) - config_str = "|".join(config_strs) - task_hash = hashlib.sha256(config_str.encode()).hexdigest()[:16] - self._task_hashes[full_task_name] = task_hash - return self._task_hashes[full_task_name] - - def get_cache_path(self, task_id: TaskID) -> Path: - """Get the file path for a specific task's cache file. - - Args: - task_id: TaskID of the task - - Returns: - Path: Path to the cache file for the given task and sample type - """ - return self.cache_dir / task_id.task_name / task_id.task_hash / f"{task_id.sampling_method.name}.parquet" - - def get_task_id(self, task_name: str, sampling_method: SamplingMethod) -> TaskID: - """Returns a unique task indentifier. Depends on the task name, - task version and parameters (from which a hash is derived), and - current sampling method (current metric we look at). - - Args: - task_name (str): Name of the task - sampling_method (SamplingMethod): Sampling used for the current metric - - Returns: - TaskID: A unique identifier for the task - """ - task_hash = self._get_task_hash(task_name) - return TaskID(task_name, task_hash, sampling_method) - - def get_sampling_method(self, sample: dict) -> str: - if len(sample.get("logprobs", [])) > 0: - return SamplingMethod.LOGPROBS - if len(sample.get("text", [])) > 0: - return SamplingMethod.GENERATIVE - return None - - def _load_sample(self, sample: pd.core.series.Series | dict) -> Union[dict, ModelResponse]: - """Load a sample from cached data based on sample type. - - Args: - sample: Raw sample data from cache, arrives as a dataframe row - - Returns: - Union[dict, ModelResponse]: Loaded sample in appropriate format for processing - """ - # If we just use the pandas dict, lists are converted to np arrays which we don't want - if isinstance(sample, pd.core.series.Series): - sample = json.loads(sample.to_json()) - return ModelResponse(**sample["sample"]) - - def _dump_sample(self, result: Union[dict, ModelResponse]) -> dict: - """Dumps the sample in the correct format for file saving - - Args: - result (Union[dict, ModelResponse]): Processed sample to save - - Returns: - dict - """ - return asdict(result) - - def get_samples_to_process_and_cache( - self, docs: List[Doc], sampling_method: SamplingMethod - ) -> Tuple[List[Doc], Set[TaskID]]: - """ - Identify which docs need processing because they are not cached yet, based on cached doc and task indices. - - Returns: - Tuple of (docs_not_cached, tasks_with_cached_samples) where - - docs_not_cached contains docs that need processing - - tasks_with_cached_samples are the tasks that have some cached samples - """ - cached_indices = self.existing_indices - - docs_not_cached = [] - tasks_with_cached_samples = set() - - for doc in docs: - task_id = self.get_task_id(doc.task_name, sampling_method) - try: - if doc.id in cached_indices[task_id]: - tasks_with_cached_samples.add(task_id) - else: - docs_not_cached.append(doc) - except KeyError: # task id or sampling method not yet there - docs_not_cached.append(doc) - - return docs_not_cached, set(tasks_with_cached_samples) - - def get_samples_from_cache( - self, docs: List[Doc], task_ids: List[TaskID] | set[TaskID], sampling_method: SamplingMethod - ) -> List[dict | ModelResponse]: - """Get cached samples for the given docs. - Warning: Assumes all docs and task_names provided are stored in cache, will fail otherwise. - - Returns: - List of cached items - """ - # Load datasets for tasks that have cached docs - task_datasets = {} - - for task_id in task_ids: - if task_id.sampling_method != sampling_method: - continue - cache_file = self.get_cache_path(task_id) - try: - dataset = load_dataset("parquet", data_files=str(cache_file), split="train") - dataset_df = dataset.to_pandas().set_index("sample_id") - task_datasets[task_id] = dataset_df - except Exception as e: - logger.warning(f"Error loading prediction cache for {str(task_id)}: {e}") - - # Build results list - results = [] - - for doc in docs: - task_id = self.get_task_id(doc.task_name, sampling_method) - row = task_datasets[task_id].loc[doc.id] - results.append(self._load_sample(row)) - - return results - - def cache_samples( # noqa C901 - self, - docs: List[Doc], - results: List[dict] | List[ModelResponse], - task_ids: list[TaskID], - sampling_method: SamplingMethod, - ): - """Store new results for samples in docs""" - if not results: - return - - # Prepare newly processed data for dataset - processed_data = {task_id: [] for task_id in task_ids} - for doc, result in zip(docs, results): - task_id = self.get_task_id(doc.task_name, sampling_method) - sample = self._dump_sample(result) - - processed_data[task_id].append({"sample_id": doc.id, "sample": sample}) - processed_data = {task_id: task_data for task_id, task_data in processed_data.items() if task_data} - - # Concatenate it with existing data and save to file - for task_id, task_data in processed_data.items(): - if task_id not in self.existing_indices.keys(): - self.existing_indices[task_id] = {} - - cache_file = self.get_cache_path(task_id) - - # Load existing data if present - existing_data = [] - existing_samples = {} - if cache_file.exists(): - try: - existing_dataset = load_dataset("parquet", data_files=str(cache_file), split="train") - existing_data = existing_dataset.to_list() - except KeyError: - logger.info(f"No data was cached for {str(task_id)}") - except Exception as e: - logger.error(f"Error loading existing prediction cache for {str(task_id)}: {e}") - - existing_samples = {(row["sample_id"], sampling_method) for row in existing_data} - if any((row["sample_id"], sampling_method) in existing_samples for row in task_data): - logger.warning( - "Unexpected behavior: You have reprocessed already cached items - we will ignore the new version." - ) - - # Merge with new data (new data overwrites existing) - # We look at id + sampling method - new_data = [row for row in task_data if (row["sample_id"], sampling_method) not in existing_samples] - all_samples = existing_data + new_data - - # Save updated dataset - dataset = Dataset.from_list(all_samples) - dataset.to_parquet(str(cache_file)) - - logger.info(f"Cached {len(all_samples)} samples of {str(task_id)} at {str(cache_file)}.") - - # Refresh cached indices after storing new samples - self.existing_indices[task_id] = [sample["sample_id"] for sample in all_samples] - - -def cached(sampling_method: SamplingMethod = None): # noqa C901 - """ - Decorator to cache method results based on Doc inputs. - - Args: - sampling_method: Sampling method to cache - - Usage: - @cached(SamplingMethod.GENERATIVE) - def greedy_until(self, docs: List[Doc], ...): - # method implementation - - Returns: - Callable: A decorator function that wraps the original function with caching functionality - """ - - def decorator(func: Callable): # noqa C901 - @functools.wraps(func) - def wrapper(self, docs: Union[Doc, List[Doc]], *args, **kwargs): # noqa C901 - docs = as_list(docs) - - # Check if caching is enabled for the model - if not hasattr(self, "_cache") or self._cache is None: - return func(self, docs, *args, **kwargs) - - cache: SampleCache = self._cache - - # Extract task names - task_ids = {cache.get_task_id(doc.task_name, sampling_method) for doc in docs} - - # 1) Identify which samples must be processed because they are not cached - docs_not_cached: List[Doc] - tasks_with_cached_samples: Set[TaskID] - docs_not_cached, tasks_with_cached_samples = cache.get_samples_to_process_and_cache(docs, sampling_method) - - # Log cache statistics - cached_count = len(docs) - len(docs_not_cached) - if cached_count > 0: - logger.info( - f"Cache: {cached_count}/{len(docs)} samples are cached for tasks {', '.join(t_id.task_name for t_id in tasks_with_cached_samples)}" - ) - - # 2) Process not cached docs and save to file - new_results = [] - if docs_not_cached: - tasks_needing_sample_processing = { - cache.get_task_id(doc.task_name, sampling_method) for doc in docs_not_cached - } - logger.info( - f"Cache: Starting to process {len(docs_not_cached)}/{len(docs)} samples (not found in cache) for tasks {','.join(str(t) for t in tasks_needing_sample_processing)}" - ) - new_results = func(self, docs_not_cached, *args, **kwargs) - - # Store new results in file cache - cache.cache_samples( - docs=docs_not_cached, - results=new_results, - task_ids=task_ids, - sampling_method=sampling_method, - ) - - # 3) Create final results by pulling from newly saved file cache - final_cached_results = cache.get_samples_from_cache(docs, task_ids, sampling_method) - - # 4) We only keep samples with the correct sampling method - final_results = [ - s for s in final_cached_results if cache.get_sampling_method(cache._dump_sample(s)) == sampling_method - ] - - if any(r is None for r in final_results): - raise ValueError("Problem while loading and aggregating items from cache.") - - return final_results - - return wrapper - - return decorator diff --git a/src/lighteval/utils/imports.py b/src/lighteval/utils/imports.py deleted file mode 100644 index 048526cbc..000000000 --- a/src/lighteval/utils/imports.py +++ /dev/null @@ -1,216 +0,0 @@ -# Copyright 2024 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import enum -import functools -import importlib -import inspect -import re -from collections import defaultdict -from functools import lru_cache -from importlib.metadata import PackageNotFoundError, metadata, version -from typing import Dict, List, Tuple - -from packaging.requirements import Requirement -from packaging.version import Version - - -# These extras should exist in the pyproject.toml file -class Extra(enum.Enum): - MULTILINGUAL = "multilingual" - EXTENDED = "extended" - TGI = "tgi" - - -@lru_cache() -def is_package_available(package: str | Requirement | Extra): - """ - Check if a package is installed in the environment. Returns True if that's the case, False otherwise. - """ - deps, deps_by_extra = required_dependencies() - - # If the package is a string, it will get the potential required version from the pyproject.toml - if isinstance(package, str): - if package not in deps: - raise RuntimeError( - f"Package {package} was tested against, but isn't specified in the pyproject.toml file. Please specify" - f"it as a potential dependency or an extra for it to be checked." - ) - package = deps[package] - - # If the specified package is an "Extra", we will iterate through each required dependency of that extra - # and their version and check their existence. - if isinstance(package, Extra): - dependencies = deps_by_extra[package.value] - return all(is_package_available(_package) for _package in dependencies) - else: - try: - installed = Version(version(package.name)) - except PackageNotFoundError: - return False - - # No version constraint → any installed version is OK - if not package.specifier: - return True - - return installed in package.specifier - - -@lru_cache() -def required_dependencies() -> Tuple[Dict[str, Requirement], Dict[str, List[Requirement]]]: - """ - Parse the pyproject.toml file and return a dictionary mapping package names to requirements. - """ - md = metadata("lighteval") - requires_dist = md.get_all("Requires-Dist") or [] - deps_by_extra = defaultdict(list) - deps = {} - - for dep in requires_dist: - extra = None - if ";" in dep: - dep, marker = dep.split(";", 1) - - # The `metadata` function prints requirements as follows - # 'vllm<0.10.2,>=0.10.0; extra == "vllm"' - # The regex searches for "extra == " in order to parse the marker. - match = re.search(r'extra\s*==\s*"(.*?)"', marker) - extra = match.group(1) if match else None - requirement = Requirement(dep.strip()) - deps_by_extra[extra].append(requirement) - deps[requirement.name] = requirement - - return deps, deps_by_extra - - -@lru_cache() -def is_multilingual_package_available(language: str): - imports = [] - packages = ["spacy", "stanza"] - if language == "vi": - packages.append("pyvi") - elif language == "zh": - packages.append("jieba") - - for package in packages: - imports.append(importlib.util.find_spec(package)) - - return all(cur_import is not None for cur_import in imports) - - -def raise_if_package_not_available(package: Requirement | Extra, *, language: str = None, object_name: str = None): - prefix = "You" if object_name is None else f"Through the use of {object_name}, you" - - if package == Extra.MULTILINGUAL and ((language is not None) or (not is_multilingual_package_available(language))): - raise ImportError(prefix + not_installed_error_message(package)[3:]) - - if not is_package_available(package): - raise ImportError(prefix + not_installed_error_message(package)[3:]) - - -def not_installed_error_message(package: Requirement) -> str: - """ - Custom error messages if need be. - """ - - if package == Extra.MULTILINGUAL.value: - return "You are trying to run an evaluation requiring multilingual capabilities. Please install the required extra: `pip install lighteval[multilingual]`" - elif package == Extra.EXTENDED.value: - return "You are trying to run an evaluation requiring additional extensions. Please install the required extra: `pip install lighteval[extended] " - elif package == "text_generation": - return "You are trying to start a text generation inference endpoint, but TGI is not present in your local environment. Please install it using pip." - elif package == "peft": - return "You are trying to use adapter weights models, for which you need `peft`, which is not available in your environment. Please install it using pip." - elif package == "openai": - return "You are trying to use an Open AI LLM as a judge, for which you need `openai`, which is not available in your environment. Please install it using pip." - - if isinstance(package, Extra): - return f"You are trying to run an evaluation requiring {package.value} capabilities. Please install the required extra: `pip install lighteval[{package.value}]`" - else: - return f"You requested the use of `{package}` for this evaluation, but it is not available in your current environment. Please install it using pip." - - -class DummyObject(type): - """ - Metaclass for the dummy objects. Any class inheriting from it will return the ImportError generated by - `requires_backend` each time a user tries to access any method of that class. - """ - - is_dummy = True - - def __getattribute__(cls, key): - if (key.startswith("_") and key != "_from_config") or key == "is_dummy" or key == "mro" or key == "call": - return super().__getattribute__(key) - - for backend in cls._backends: - raise_if_package_not_available(backend) - - return super().__getattribute__(key) - - -def parse_specified_backends(specified_backends): - requirements, _ = required_dependencies() - applied_backends = [] - - for backend in specified_backends: - if isinstance(backend, Extra): - applied_backends.append(backend if isinstance(backend, Extra) else requirements[backend]) - elif backend not in requirements: - raise RuntimeError( - "A dependency was specified with @requires, but it is not defined in the possible dependencies " - f"defined in the pyproject.toml: `{backend}`." - f"" - f"If editing the pyproject.toml to add a new dependency, remember to reinstall lighteval for the" - f"update to take effect." - ) - else: - applied_backends.append(requirements[backend]) - - return applied_backends - - -def requires(*specified_backends): - """ - Decorator to raise an ImportError if the decorated object (function or class) requires a dependency - which is not installed. - """ - - applied_backends = parse_specified_backends(specified_backends) - - def inner_fn(_object): - _object._backends = applied_backends - - if inspect.isclass(_object): - - class Placeholder(metaclass=DummyObject): - _backends = applied_backends - - def __init__(self, *args, **kwargs): - for backend in self._backends: - raise_if_package_not_available(backend, object_name=_object.__name__) - - Placeholder.__name__ = _object.__name__ - Placeholder.__module__ = _object.__module__ - - return _object if all(is_package_available(backend) for backend in applied_backends) else Placeholder - else: - - @functools.wraps(_object) - def wrapper(*args, **kwargs): - for backend in _object._backends: - raise_if_package_not_available(backend, object_name=_object.__name__) - return _object(*args, **kwargs) - - return wrapper - - return inner_fn diff --git a/src/lighteval/utils/language.py b/src/lighteval/utils/language.py deleted file mode 100644 index f2f98ad37..000000000 --- a/src/lighteval/utils/language.py +++ /dev/null @@ -1,385 +0,0 @@ -# Copyright 2024 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from enum import Enum - - -class Language(Enum): - ACEHNESE = "ace" - AFRIKAANS = "afr" - AKAN = "aka" - ALBANIAN = "sqi" - AMHARIC = "amh" - ARABIC = "ara" - ARMENIAN = "hye" - ASSAMESE = "asm" - ASTURIAN = "ast" - AWADHI = "awa" - AYACUCHO_QUECHUA = "quy" - AZERBAIJANI = "aze" - BALINESE = "ban" - BAMBARA = "bam" - BANJAR = "bjn" - BASHKIR = "bak" - BASQUE = "eus" - BELARUSIAN = "bel" - BEMBA = "bem" - BENGALI = "ben" - BHOJPURI = "bho" - BIHARI = "bih" # Deprecated - BOSNIAN = "bos" - BRETON = "bre" - BUGINESE = "bug" - BULGARIAN = "bul" - BURMESE = "mya" - CATALAN = "cat" - CEBUANO = "ceb" - CENTRAL_ATLAS_TAMAZIGHT = "tzm" - CENTRAL_AYMARA = "ayr" - CENTRAL_KANURI = "knc" - CENTRAL_KURDISH = "ckb" - CHHATTISGARHI = "hne" - CHINESE = "zho" - CHOKWE = "cjk" - CRIMEAN_TATAR = "crh" - CROATIAN = "hrv" - CZECH = "ces" - DANISH = "dan" - DARI = "prs" - DIVEHI = "div" - DUTCH = "nld" - DYULA = "dyu" - DZONGKHA = "dzo" - EASTERN_PANJABI = "pan" - EASTERN_YIDDISH = "ydd" - EGYPTIAN_ARABIC = "arz" - ENGLISH = "eng" - ESPERANTO = "epo" - ESTONIAN = "est" - EWE = "ewe" - FAROESE = "fao" - FIJIAN = "fij" - FINNISH = "fin" - FON = "fon" - FRENCH = "fra" - FRIULIAN = "fur" - GALICIAN = "glg" - GANDA = "lug" - GEORGIAN = "kat" - GERMAN = "deu" - GREEK = "ell" - GUARANI = "grn" - GUJARATI = "guj" - HAITIAN = "hti" - HAITIAN_CREOLE = "hat" - HALH_MONGOLIAN = "khk" - HAUSA = "hau" - HEBREW = "heb" - HINDI = "hin" - HUNGARIAN = "hun" - ICELANDIC = "isl" - IGBO = "ibo" - ILOCANO = "ilo" - INDONESIAN = "ind" - IRISH = "gle" - ITALIAN = "ita" - JAPANESE = "jpn" - JAVANESE = "jav" - JINGPHO = "kac" - KABIYE = "kbp" - KABUVERDIANU = "kea" - KABYLE = "kab" - KAMBA = "kam" - KANNADA = "kan" - KASHMIRI = "kas" - KAZAKH = "kaz" - KHMER = "khm" - KIKONGO = "kon" - KIKUYU = "kik" - KIMBUNDU = "kmb" - KINYARWANDA = "kin" - KIRGHIZ = "kir" - KOREAN = "kor" - KURDISH = "kur" - KYRGYZ = "kir" - LAO = "lao" - LATGALIAN = "ltg" - LATIN = "lat" - LATVIAN = "lav" - LIGURIAN = "lij" - LIMBURGISH = "lim" - LINGALA = "lin" - LITHUANIAN = "lit" - LOMBARD = "lmo" - LUBA_KASAI = "lua" - LUO = "luo" - LUXEMBOURGISH = "ltz" - MACEDONIAN = "mkd" - MAGAHI = "mag" - MAITHILI = "mai" - MALAGASY = "mlg" - MALAY = "msa" - MALAYALAM = "mal" - MALTESE = "mlt" - MAORI = "mri" - MARATHI = "mar" - MEITEI = "mni" - MESOPOTAMIAN_ARABIC = "acm" - MINANGKABAU = "min" - MIZO = "lus" - MODERN_STANDARD_ARABIC = "arb" - MOROCCAN_ARABIC = "ary" - MOSSI = "mos" - NAJDI_ARABIC = "ars" - NEPALI = "nep" - NIGERIAN_FULFULDE = "fuv" - NORTHERN_KURDISH = "kmr" - NORTHERN_SOTHO = "nso" - NORTHERN_UZBEK = "uzn" - NORTH_AZERBAIJANI = "azj" - NORTH_LEVANTINE_ARABIC = "apc" - NORWEGIAN = "nor" - NORWEGIAN_BOKMAL = "nob" - NORWEGIAN_NYNORSK = "nno" - NUER = "nus" - NYANJA = "nya" - OCCITAN = "oci" - ODIA = "ory" - ORIYA = "ori" - PANGASINAN = "pag" - PAPIAMENTO = "pap" - PASHTO = "pus" - PERSIAN = "fas" - PLATEAU_MALAGASY = "plt" - POLISH = "pol" - PORTUGUESE = "por" - PUNJABI = "pan" - QUECHUA = "que" - ROMANIAN = "ron" - RUNDI = "run" - RUSSIAN = "rus" - SAMOAN = "smo" - SANGO = "sag" - SANSKRIT = "san" - SANTALI = "sat" - SARDINIAN = "srd" - SCOTTISH_GAELIC = "gla" - SERBIAN = "srp" - SERBOCROATIAN = "hbs" # Deprecated - SHAN = "shn" - SHONA = "sna" - SICILIAN = "scn" - SILESIAN = "szl" - SINDHI = "snd" - SINHALA = "sin" - SLOVAK = "slk" - SLOVENIAN = "slv" - SOMALI = "som" - SORANI = "ckb" - SOUTHERN_PASHTO = "pbt" - SOUTHERN_SOTHO = "sot" - SOUTHWESTERN_DINKA = "dik" - SOUTH_AZERBAIJANI = "azb" - SOUTH_LEVANTINE_ARABIC = "ajp" - SPANISH = "spa" - STANDARD_LATVIAN = "lvs" - STANDARD_MALAY = "zsm" - STANDARD_TIBETAN = "bod" - SUNDANESE = "sun" - SWAHILI = "swa" - SWATI = "ssw" - SWEDISH = "swe" - TAGALOG = "tgl" - TAJIK = "tgk" - TAMASHEQ = "taq" - TAMIL = "tam" - TATAR = "tat" - TAIZZI_ADENI_ARABIC = "acq" - TELUGU = "tel" - THAI = "tha" - TIGRINYA = "tir" - TOK_PISIN = "tpi" - TOSK_ALBANIAN = "als" - TSONGA = "tso" - TSWANA = "tsn" - TUMBUKA = "tum" - TUNISIAN_ARABIC = "aeb" - TURKISH = "tur" - TURKMEN = "tuk" - TWI = "twi" - UDMURT = "udm" - UKRAINIAN = "ukr" - UMBUNDU = "umb" - URDU = "urd" - UYGHUR = "uig" - UZBEK = "uzb" - VENETIAN = "vec" - VIETNAMESE = "vie" - WAR = "war" - WARAY = "war" - WELSH = "cym" - WESTERN_FRISIAN = "fry" - WESTERN_PERSIAN = "pes" - WEST_CENTRAL_OROMO = "gaz" - WOLOF = "wol" - XHOSA = "xho" - YIDDISH = "yid" - YORUBA = "yor" - YUE_CHINESE = "yue" - ZULU = "zul" - - -# This mapping was created for beleble, it converts iso_639_3 individual codes to iso_639_3 macro codes -# However it requires iso639-lang package and I don't see a point installing it just for this mapping -# Code to generate: -# ```` -# from langcodes import Language -# from iso639 import Lang - -# dst = get_dataset_config_names("facebook/belebele") -# output = {} -# for i in dst: -# lang_old = Lang(Language.get(i).language) -# lang = lang_old.macro() if lang_old.macro() else lang_old -# output[lang_old.pt3] = lang.pt3 -# ``` - - -def manage_duplicate_language_codes(langcode): - if langcode == "npi": # Nepali - langcode = "nep" - if langcode == "swh": # Swahili - langcode = "swa" - return langcode - - -iso_639_3_ind_to_iso_639_3_macro = { - "acm": Language.ARABIC, - "arz": Language.ARABIC, - "ceb": Language.CEBUANO, - "fin": Language.FINNISH, - "hin": Language.HINDI, - "ita": Language.ITALIAN, - "khm": Language.KHMER, - "lvs": Language.LATVIAN, - "npi": Language.NEPALI, - "pol": Language.POLISH, - "slv": Language.SLOVENIAN, - "swe": Language.SWEDISH, - # 'tso': Language.TSONGA, - # 'xho': Language.XHOSA, - "afr": Language.AFRIKAANS, - "asm": Language.ASSAMESE, - "ces": Language.CZECH, - "fra": Language.FRENCH, - "jav": Language.JAVANESE, - # 'kin': Language.KINYARWANDA, - "mal": Language.MALAYALAM, - "por": Language.PORTUGUESE, - # 'sna': Language.SHONA, - "swh": Language.SWAHILI, - "tur": Language.TURKISH, - "yor": Language.YORUBA, - "als": Language.ALBANIAN, - "azj": Language.AZERBAIJANI, - "ckb": Language.KURDISH, - # 'fuv': Language.FULAH, - "hrv": Language.CROATIAN, - "jpn": Language.JAPANESE, - "kir": Language.KIRGHIZ, - "mar": Language.MARATHI, - # 'nso': Language.NORTHERN_SOTHO, - "snd": Language.SINDHI, - "tam": Language.TAMIL, - "ukr": Language.UKRAINIAN, - "zho": Language.CHINESE, - "amh": Language.AMHARIC, - # 'bam': Language.BAMBARA, - "dan": Language.DANISH, - # 'gaz': Language.OROMO, - "hun": Language.HUNGARIAN, - # 'kac': Language.KACHIN, - "kor": Language.KOREAN, - "mkd": Language.MACEDONIAN, - # 'nya': Language.CHICHEWA, - "ron": Language.ROMANIAN, - "som": Language.SOMALI, - "tel": Language.TELUGU, - "urd": Language.URDU, - "apc": Language.ARABIC, - "ben": Language.BENGALI, - "deu": Language.GERMAN, - # 'grn': Language.GUARANI, - "hye": Language.ARMENIAN, - "kan": Language.KANNADA, - "lao": Language.LAO, - "mlt": Language.MALTESE, - "ory": Language.ORIYA, - "rus": Language.RUSSIAN, - # 'sot': Language.SOUTHERN_SOTHO, - "tgk": Language.TAJIK, - "zsm": Language.MALAY, - "arb": Language.ARABIC, - "ell": Language.GREEK, - "guj": Language.GUJARATI, - # 'ibo': Language.IGBO, - "kat": Language.GEORGIAN, - # 'lin': Language.LINGALA, - # 'mri': Language.MAORI, - "pan": Language.PUNJABI, - "shn": Language.SHAN, - "spa": Language.SPANISH, - "fil": Language.TAGALOG, - "uzn": Language.UZBEK, - # 'zul': Language.ZULU, - # 'bod': Language.TIBETAN, - "eng": Language.ENGLISH, - "hat": Language.HAITIAN, - # 'ilo': Language.ILOCANO, - "kaz": Language.KAZAKH, - "lit": Language.LITHUANIAN, - "mya": Language.BURMESE, - "pbt": Language.PASHTO, - "sin": Language.SINHALA, - "srp": Language.SERBIAN, - "tha": Language.THAI, - "vie": Language.VIETNAMESE, - "ars": Language.ARABIC, - "bul": Language.BULGARIAN, - "est": Language.ESTONIAN, - "udm": Language.UDMURT, - # 'hau': Language.HAUSA, - "ind": Language.INDONESIAN, - # 'kea': Language.KABUVERDIANU, - # 'lug': Language.GANDA, - "nld": Language.DUTCH, - "pes": Language.PERSIAN, - # 'ssw': Language.SWATI, - # 'tir': Language.TIGRINYA, - "war": Language.WAR, - "ary": Language.ARABIC, - "cat": Language.CATALAN, - "eus": Language.BASQUE, - "que": Language.QUECHUA, - "heb": Language.HEBREW, - "isl": Language.ICELANDIC, - # 'khk': Language.MONGOLIAN, - # 'luo': Language.LUO, - "nob": Language.NORWEGIAN, - "plt": Language.MALAGASY, - "slk": Language.SLOVAK, - # 'sun': Language.SUNDANESE, - # 'tsn': Language.TSWANA, - # 'wol': Language.WOLOF -} diff --git a/src/lighteval/utils/parallelism.py b/src/lighteval/utils/parallelism.py deleted file mode 100644 index cedf1a07a..000000000 --- a/src/lighteval/utils/parallelism.py +++ /dev/null @@ -1,161 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import functools -import gc -import inspect -import logging - -import torch - -from lighteval.utils.imports import raise_if_package_not_available - - -logger = logging.getLogger(__name__) - - -def should_reduce_batch_size(exception: Exception) -> bool: - """Checks if `exception` relates to CUDA out-of-memory, CUDNN not supported, or CPU out-of-memory - - Args: - exception (`Exception`): - An exception - - Returns: - bool: True if the exception is related to memory issues that can be resolved by reducing batch size - """ - _statements = [ - "CUDA out of memory.", # CUDA OOM - "cuDNN error: CUDNN_STATUS_NOT_SUPPORTED.", # CUDNN SNAFU - "DefaultCPUAllocator: can't allocate memory", # CPU OOM - "HIP out of memory", # ROCM OOM - ] - if isinstance(exception, RuntimeError) and len(exception.args) == 1: - return any(err in exception.args[0] for err in _statements) - return False - - -def find_executable_batch_size(function: callable = None, starting_batch_size: int = 128): - """A basic decorator that will try to execute `function`. If it fails from exceptions related to out-of-memory or - CUDNN, the batch size is cut in half and passed to `function` - - `function` must take in a `batch_size` parameter as its first argument. - - Args: - function (`callable`, *optional*): - A function to wrap - starting_batch_size (`int`, *optional*): - The batch size to try and fit into memory - - Example: - ```python - >>> from lighteval.utils_parallelism import find_executable_batch_size - - - >>> @find_executable_batch_size(starting_batch_size=128) - ... def train(batch_size, model, optimizer): - ... ... - - - >>> train(model, optimizer) - ``` - - Returns: - Callable: A decorator function that automatically adjusts batch size to fit in memory - """ - if function is None: - return functools.partial(find_executable_batch_size, starting_batch_size=starting_batch_size) - - batch_size = starting_batch_size - - def decorator(*args, **kwargs): - nonlocal batch_size - gc.collect() - torch.cuda.empty_cache() - params = list(inspect.signature(function).parameters.keys()) - # Guard against user error - if len(params) < (len(args) + 1): - arg_str = ", ".join([f"{arg}={value}" for arg, value in zip(params[1:], args[1:])]) - raise TypeError( - f"Batch size was passed into `{function.__name__}` as the first argument when called." - f"Remove this as the decorator already does so: `{function.__name__}({arg_str})`" - ) - while True: - if batch_size == 0: - raise RuntimeError("No executable batch size found, reached zero.") - try: - return function(batch_size, *args, **kwargs) - except Exception as e: - if should_reduce_batch_size(e): - gc.collect() - torch.cuda.empty_cache() - batch_size //= 2 - else: - raise - - return decorator - - -def test_all_gather(accelerator=None, parallel_context=None): - """Test the gather operation in a parallel setup. - - Args: - accelerator (Optional): The accelerator object used for parallelism. - parallel_context (Optional): The parallel context object used for parallelism. - """ - if accelerator: - raise_if_package_not_available("accelerate") - logger.info("Test gather tensor") - test_tensor: torch.Tensor = torch.tensor([accelerator.process_index], device=accelerator.device) - gathered_tensor: torch.Tensor = accelerator.gather(test_tensor) - logger.info(f"gathered_tensor {gathered_tensor}, should be {list(range(accelerator.num_processes))}") - accelerator.wait_for_everyone() - elif parallel_context: - raise_if_package_not_available("nanotron") - from nanotron import distributed as dist - from nanotron import logging - - logger.info("Test gather tensor") - # Do a first NCCL sync to warmup and try to avoid Timeout after model/data loading - logging.log_rank( - f"[TEST] Running NCCL sync for ranks {list(range(parallel_context.world_pg.size()))}", - logger=logger, - level=logging.WARNING, - group=parallel_context.dp_pg, - rank=0, - ) - test_tensor = torch.tensor([dist.get_rank(parallel_context.world_pg)], device=torch.device("cuda")) - test_tensor_list = [torch.zeros_like(test_tensor) for _ in range(parallel_context.world_pg.size())] - dist.all_gather(test_tensor_list, test_tensor, group=parallel_context.world_pg, async_op=False) - dist.barrier() - logging.log_rank( - f"[TEST] NCCL sync for ranks {[t.item() for t in test_tensor_list]}", - logger=logger, - level=logging.WARNING, - group=parallel_context.dp_pg, - rank=0, - ) - - del test_tensor_list - del test_tensor - else: - logger.info("Not running in a parallel setup, nothing to test") diff --git a/src/lighteval/utils/timeout.py b/src/lighteval/utils/timeout.py deleted file mode 100644 index 0b9e248c2..000000000 --- a/src/lighteval/utils/timeout.py +++ /dev/null @@ -1,98 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import os - - -def timeout(timeout_seconds: int = 10): # noqa: C901 - """A decorator that applies a timeout to the decorated function. - - Args: - timeout_seconds (int): Number of seconds before timing out the decorated function. - Defaults to 10 seconds. - - Notes: - On Unix systems, uses a signal-based alarm approach which is more efficient as it doesn't require spawning a new process. - On Windows systems, uses a multiprocessing-based approach since signal.alarm is not available. This will incur a huge performance penalty. - - Returns: - Callable: A decorator function that wraps the original function with timeout functionality - """ - if os.name == "posix": - # Unix-like approach: signal.alarm - import signal - - def decorator(func): - def handler(signum, frame): - raise TimeoutError("Operation timed out!") - - def wrapper(*args, **kwargs): - old_handler = signal.getsignal(signal.SIGALRM) - signal.signal(signal.SIGALRM, handler) - signal.alarm(timeout_seconds) - try: - return func(*args, **kwargs) - finally: - # Cancel the alarm and restore previous handler - signal.alarm(0) - signal.signal(signal.SIGALRM, old_handler) - - return wrapper - - return decorator - - else: - # Windows approach: use multiprocessing - from multiprocessing import Process, Queue - - def decorator(func): - def wrapper(*args, **kwargs): - q = Queue() - - def run_func(q, args, kwargs): - try: - result = func(*args, **kwargs) - q.put((True, result)) - except Exception as e: - q.put((False, e)) - - p = Process(target=run_func, args=(q, args, kwargs)) - p.start() - p.join(timeout_seconds) - - if p.is_alive(): - # Timeout: Terminate the process - p.terminate() - p.join() - raise TimeoutError("Operation timed out!") - - # If we got here, the process completed in time. - success, value = q.get() - if success: - return value - else: - # The child raised an exception; re-raise it here - raise value - - return wrapper - - return decorator diff --git a/src/lighteval/utils/utils.py b/src/lighteval/utils/utils.py deleted file mode 100644 index 3ab5976d8..000000000 --- a/src/lighteval/utils/utils.py +++ /dev/null @@ -1,317 +0,0 @@ -# Copyright 2023 The HuggingFace Team. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from dataclasses import asdict, is_dataclass -from typing import TypeVar, Union - -import numpy as np -from pytablewriter import MarkdownTableWriter - - -def flatten_dict(nested: dict, sep: str = "/") -> dict: - """Flattens a nested dictionary structure into a single-level dictionary. - - Recursively traverses nested dictionaries, lists, and tuples, concatenating keys with a separator - to create unique keys for each value. Handles numpy arrays and sanitizes markdown-incompatible characters. - - Args: - nested (dict): The nested dictionary structure to flatten. - sep (str, optional): The separator to use between nested keys. Defaults to "/". - - Returns: - dict: A flattened dictionary where all values are at the root level and keys indicate - the original nesting structure using the separator. - - Examples: - >>> flatten_dict({'a': {'b': 1, 'c': [{'d': 2}, {'e': 3}]}}) - {'a/b': 1, 'a/c/0/d': 2, 'a/c/1/e': 3} - """ - - def clean_markdown(v: str) -> str: - return v.replace("|", "_").replace("\n", "_") if isinstance(v, str) else v # Need this for markdown - - def rec(nest: dict, prefix: str, into: dict): - for k, v in sorted(nest.items()): - # if sep in k: - # raise ValueError(f"separator '{sep}' not allowed to be in key '{k}'") - if isinstance(v, dict): - rec(v, prefix + k + sep, into) - elif isinstance(v, (list, tuple)): - for i, vv in enumerate(v): - if isinstance(vv, dict): - rec(vv, prefix + k + sep + str(i) + sep, into) - else: - vv = ( - vv.replace("|", "_").replace("\n", "_") if isinstance(vv, str) else vv - ) # Need this for markdown - into[prefix + k + sep + str(i)] = vv.tolist() if isinstance(vv, np.ndarray) else vv - elif isinstance(v, np.ndarray): - into[prefix + k + sep + str(i)] = v.tolist() - else: - v = clean_markdown(v) - into[prefix + k] = v - - flat = {} - rec(nested, "", flat) - return flat - - -def clean_s3_links(value: str) -> str: - """Converts S3 bucket links to clickable HTML links for better display in result tables. - - Takes an S3 URI and converts it to a clickable HTML link that opens the AWS S3 console - at the specific bucket and prefix location. This is particularly useful for nanotron models - and other S3-hosted model artifacts. - - Args: - value (str): The S3 URI to convert (e.g., "s3://my-bucket/path/to/model"). - - Returns: - str: HTML anchor tag with the original S3 URI as display text and a link to the AWS S3 console. - - Examples: - >>> clean_s3_links("s3://my-bucket/models/gpt2") - ' s3://my-bucket/models/gpt2 ' - """ - s3_bucket, s3_prefix = str(value).replace("s3://", "").split("/", maxsplit=1) - if not s3_prefix.endswith("/"): - s3_prefix += "/" - link_str = f"https://s3.console.aws.amazon.com/s3/buckets/{s3_bucket}?prefix={s3_prefix}" - value = f' {value} ' - return value - - -def obj_to_markdown(obj, convert_s3_links: bool = True) -> str: - """Converts a dataclass object or dictionary to a readable markdown table for logging. - - Flattens the object structure and creates a markdown table with key-value pairs. - Optionally converts S3 links to clickable HTML links for better display. - - Args: - obj: A dataclass object or dictionary to convert to markdown. - convert_s3_links (bool, optional): Whether to convert S3 URIs to clickable links. - Defaults to True. - - Returns: - str: A markdown-formatted table string with the object's key-value pairs. - - Examples: - >>> from dataclasses import dataclass - >>> @dataclass - ... class Config: - ... model_name: str = "gpt2" - ... batch_size: int = 32 - >>> obj_to_markdown(Config()) - '| Key | Value |\n|-----|-------|\n| model_name | gpt2 |\n| batch_size | 32 |' - """ - from pytablewriter import MarkdownTableWriter - - if is_dataclass(obj): - obj = asdict(obj) - config_dict = flatten_dict(obj) - - md_writer = MarkdownTableWriter() - md_writer.headers = ["Key", "Value"] - - values = [] - for key, value in config_dict.items(): - if convert_s3_links and "s3://" in str(value): - value = clean_s3_links(value) - values.append([key, value]) - md_writer.value_matrix = values - - return md_writer.dumps() - - -def sanitize_numpy(example_dict: dict) -> dict: - """Sanitizes a dictionary by converting numpy generic types to their corresponding Python types. - - Numpy generic types (like np.int64, np.float32, etc.) can cause issues with JSON serialization - and other operations. This function converts them to standard Python types. - - Args: - example_dict (dict): The dictionary to be sanitized. - - Returns: - dict: The sanitized dictionary with numpy generic types converted to Python types. - - Examples: - >>> import numpy as np - >>> data = {'score': np.float32(0.95), 'count': np.int64(42)} - >>> sanitize_numpy(data) - {'score': 0.95, 'count': 42} - """ - output_dict = {} - for k, v in example_dict.items(): - if isinstance(v, np.generic): - output_dict[k] = v.item() - else: - output_dict[k] = v - return output_dict - - -ListLikeTypeVar = TypeVar("ListLikeTypeVar") -ListLike = list[ListLikeTypeVar] | tuple[ListLikeTypeVar, ...] - - -ElementType = TypeVar("ElementType") - - -def as_list(item: ListLike[ElementType] | ElementType) -> list[ElementType]: - """Converts the given item into a list. - - If the item is already a list, it is returned as is. If the item is a tuple, - it is converted into a list. Otherwise, the item is wrapped in a list. - - Args: - item: The item to be converted. Can be a list, tuple, or any other type. - - Returns: - list: The converted list. - - Examples: - >>> as_list([1, 2, 3]) - [1, 2, 3] - >>> as_list((1, 2, 3)) - [1, 2, 3] - >>> as_list("single_item") - ['single_item'] - >>> as_list(42) - [42] - """ - if isinstance(item, list): - return item - - elif isinstance(item, tuple): - return list(item) - - return [item] - - -def flatten(item: list[Union[list, str]]) -> list[str]: - """Flattens a nested list of strings into a single flat list. - - Recursively flattens nested lists, extracting all string elements into a single-level list. - Non-list elements are treated as strings and added directly. - - Args: - item: The nested list to be flattened. Can contain strings and nested lists. - - Returns: - list[str]: The flattened list of strings. - - Examples: - >>> flatten([["a", "b"], "c", ["d", ["e", "f"]]]) - ['a', 'b', 'c', 'd', 'e', 'f'] - >>> flatten(["simple", "list"]) - ['simple', 'list'] - """ - flat_item = [] - for sub_item in item: - flat_item.extend(sub_item) if isinstance(sub_item, list) else flat_item.append(sub_item) - return flat_item - - -def make_results_table(result_dict): - """Generates a markdown table from evaluation results. - - Creates a formatted markdown table displaying task results with metrics, values, - and standard errors. The table includes columns for task name, version, metric, - value, and standard error. - - Args: - result_dict (dict): Dictionary containing evaluation results with the structure: - - 'results': Dict mapping task names to metric dictionaries - - 'versions': Dict mapping task names to version strings - - Returns: - str: A markdown-formatted table string displaying the results. - - Examples: - >>> results = { - ... 'results': { - ... 'squad': {'accuracy': 0.85, 'accuracy_stderr': 0.02}, - ... 'glue': {'f1': 0.92, 'f1_stderr': 0.01} - ... }, - ... 'versions': {'squad': 'v2.0', 'glue': 'v1.0'} - ... } - >>> table = make_results_table(results) - # Returns markdown table with task, version, metric, value, ±, stderr columns - """ - md_writer = MarkdownTableWriter() - md_writer.headers = ["Task", "Version", "Metric", "Value", "", "Stderr"] - - values = [] - - for k in sorted(result_dict["results"].keys()): - dic = result_dict["results"][k] - version = result_dict["versions"][k] if k in result_dict["versions"] else "" - for m, v in dic.items(): - if m.endswith("_stderr"): - continue - - if m + "_stderr" in dic: - se = dic[m + "_stderr"] - values.append([k, version, m, "%.4f" % v, "±", "%.4f" % se]) - else: - values.append([k, version, m, "%.4f" % v, "", ""]) - k = "" - version = "" - md_writer.value_matrix = values - - return md_writer.dumps() - - -def safe_divide(numerator: np.ndarray, denominator: float, default_value: float = 0.0) -> np.ndarray: - return np.where(denominator != 0, numerator / denominator, default_value) - - -def remove_reasoning_tags(text: str, tag_pairs: list[tuple[str, str]]) -> str: - """Removes all instances of reasoning tag pairs from text. - - Iteratively removes content between specified start and end tag pairs. - This is useful for cleaning model outputs that contain reasoning sections - that should be excluded from evaluation. - - See: https://github.com/huggingface/lighteval/issues/790 - - Args: - text (str): The input text containing reasoning tags to remove. - tag_pairs (list[tuple[str, str]]): List of (start_tag, end_tag) pairs to remove. - - Returns: - str: The text with all reasoning tag content removed. - - Examples: - >>> text = " Reasoning section Answer section" - >>> tag_pairs = [("", "")] - >>> remove_reasoning_tags(text, tag_pairs) - ' Answer section' - - >>> text = "Step 1AnswerStep 2" - >>> tag_pairs = [("", "")] - >>> remove_reasoning_tags(text, tag_pairs) - 'Answer' - """ - result = text - - for start_tag, end_tag in tag_pairs: - while start_tag in result and end_tag in result: - start = result.find(start_tag) - end = result.find(end_tag, start) - if start != -1 and end != -1: - result = result[:start] + result[end + len(end_tag) :] - else: - break - - return result diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index a732db8d0..000000000 --- a/tests/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index a568cc130..000000000 --- a/tests/conftest.py +++ /dev/null @@ -1,23 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -import pytest - - -def pytest_addoption(parser): - parser.addoption("--runslow", action="store_true", default=False, help="run slow tests") - - -def pytest_configure(config): - config.addinivalue_line("markers", "slow: mark test as slow to run") - - -def pytest_collection_modifyitems(config, items): - if config.getoption("--runslow"): - # --runslow given in cli: do not skip slow tests - return - skip_slow = pytest.mark.skip(reason="need --runslow option to run") - for item in items: - if "slow" in item.keywords: - item.add_marker(skip_slow) diff --git a/tests/fixtures.py b/tests/fixtures.py deleted file mode 100644 index d14e542fb..000000000 --- a/tests/fixtures.py +++ /dev/null @@ -1,60 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -import os - -import pytest -from huggingface_hub import HfApi -from huggingface_hub.hf_api import DatasetInfo - - -TESTING_EMPTY_HF_ORG_ID = "lighteval-tests" - - -@pytest.fixture -def testing_empty_hf_org_id(org_id: str = TESTING_EMPTY_HF_ORG_ID): - old_token = os.getenv("HF_TOKEN") - os.environ["HF_TOKEN"] = os.getenv("HF_TEST_TOKEN") or "" - - def list_repos(org_id: str): - return list(hf_api.list_models(author=org_id)) + list(hf_api.list_datasets(author=org_id)) - - def clean_repos(org_id: str): - repos = list_repos(org_id) - for repo in repos: - hf_api.delete_repo(repo.id, repo_type="dataset" if isinstance(repo, DatasetInfo) else "model") - - hf_api = HfApi() - # Remove all repositories in the HF org - clean_repos(org_id) - - # Verify that all repositories have been removed - remaining_repos = list_repos(org_id) - assert len(remaining_repos) == 0, f"Expected 0 repositories, but found {len(remaining_repos)}" - - yield org_id - - # Clean up: recreate any necessary default repositories after the test - # This step is optional and depends on your specific needs - clean_repos(org_id) - os.environ["HF_TOKEN"] = old_token if old_token is not None else "" diff --git a/tests/reference_details/Qwen2.5-VL-3B-Instruct-vlm/details_mmmu_pro:standard-4|0_2025-11-05T15-23-34.026089.parquet b/tests/reference_details/Qwen2.5-VL-3B-Instruct-vlm/details_mmmu_pro:standard-4|0_2025-11-05T15-23-34.026089.parquet deleted file mode 100644 index 4b71bbcbe..000000000 --- a/tests/reference_details/Qwen2.5-VL-3B-Instruct-vlm/details_mmmu_pro:standard-4|0_2025-11-05T15-23-34.026089.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5eda1dbcf8c9604005ce8c27239a57e5f41f852dbd3da13656d94b01b325f16c -size 11538690 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:aqua-rat|0_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:aqua-rat|0_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index ebaf64fb5..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:aqua-rat|0_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ef968329bee498b3387ec8df3677ca9bbac72e90599efbe7f78db23f4227b2f6 -size 21785 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:logiqa-en|0_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:logiqa-en|0_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index e5b19fb8a..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:logiqa-en|0_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5bb00dd8c872d95dd5b2999d788ece8c34d43b5b5ea4ef8f0859ba535d7b8cbf -size 34021 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:lsat-ar|0_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:lsat-ar|0_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index 78aae23ef..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:lsat-ar|0_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:161eb467544ca6273231945067a8d70aedb0e4e6c3eba4e8b40532cd1c37e6b0 -size 30662 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:lsat-lr|0_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:lsat-lr|0_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index f8376974b..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:lsat-lr|0_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:da3a3dcc9ae24c6f3bc80f0ce72b64d09a5ab19d7803b168a8f1ad540f7f66c1 -size 39332 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:lsat-rc|0_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:lsat-rc|0_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index 7d12a10a2..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:lsat-rc|0_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:fb3787ff3e796b49199e8d438a30ce768438b6d4fc5df5941e393bfd1fdf2ae6 -size 74124 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:sat-en-without-passage|0_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:sat-en-without-passage|0_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index b0eaaad4b..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:sat-en-without-passage|0_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:47204bfa1d6843f06ef6c08bb66d85adceab6457295f03303b7cd39bc7e4dd37 -size 25864 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:sat-en|0_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:sat-en|0_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index 4fe3bfab2..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_agieval:sat-en|0_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8a20696f4036b3a6e2b41b309f405d9a5445c4573150fc7e7bec15f28fd77bdf -size 72441 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_arc:challenge|25_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_arc:challenge|25_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index b35987316..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_arc:challenge|25_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:803a445f6d563c2d2097df7c7772cd81b3df0bacc9e702caaab5f0dde7fe5b25 -size 87624 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:causal_judgment|3_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:causal_judgment|3_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index 41817317a..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:causal_judgment|3_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:eb844c74f574b377e4b27110dbdf0c28c227a96f4e8d1c0eac52578f4608bc49 -size 47558 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:date_understanding|3_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:date_understanding|3_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index 3935aff9b..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:date_understanding|3_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b5ce79da0c3657667830882fa28ce623cb463bf5fb3c5e1367d6a5c13c480973 -size 30006 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:disambiguation_qa|3_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:disambiguation_qa|3_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index a11330161..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:disambiguation_qa|3_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4d4919aa444d52a1589883329eb3fdbb583b029a6213d4af13aa17c11a835399 -size 30932 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:geometric_shapes|3_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:geometric_shapes|3_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index 6cba141eb..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:geometric_shapes|3_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0dffe2c495874fa53e0289b2307161107c54e9d15c9a8aa39016c990f7d62f8f -size 32464 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:logical_deduction_five_objects|3_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:logical_deduction_five_objects|3_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index 4c2a135e0..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:logical_deduction_five_objects|3_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1d10ce12e8f76b5ce3113273124e7683e5c5bddde6063cd3cbf25d495cffa6ba -size 34653 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:logical_deduction_seven_objects|3_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:logical_deduction_seven_objects|3_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index 6f025304a..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:logical_deduction_seven_objects|3_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e15bcad77e0453d7e987b4bf5216639b625f9df63341dfce4246dab88b87ca35 -size 38176 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:movie_recommendation|3_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:movie_recommendation|3_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index b3d0779da..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:movie_recommendation|3_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:64975666dfc61cd3a3a515a88134775c0f90cff1e1b9120a8ab9c8861c68bb99 -size 29221 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:navigate|3_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:navigate|3_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index 07fe47bc6..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:navigate|3_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6baa807a3c85b3ca43708f5810f3898dd517db806ce4106124d4913b0fcea8b0 -size 28834 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:ruin_names|3_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:ruin_names|3_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index 781634333..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:ruin_names|3_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7fe3c64179df68407c6fe6c6ad75b3f4a83a1edd286ca81da3fe96bcb5b21e9b -size 27971 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:salient_translation_error_detection|3_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:salient_translation_error_detection|3_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index eeb8e003c..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:salient_translation_error_detection|3_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e34151ea0415cb442b47d334448abf127c8f1747da78a5a9977ff78ed2d831b5 -size 49337 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:snarks|3_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:snarks|3_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index 3043b86ed..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:snarks|3_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:13659b3b6bcd5c2744fc3b33d8752589a1f6c52b2ed8ee17c6a3a4f28cd46908 -size 29149 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:temporal_sequences|3_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:temporal_sequences|3_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index 9350b676d..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:temporal_sequences|3_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:23ffe95306670d3a737b30bf34866734dcba717742011a424cc0230377f52363 -size 34393 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:tracking_shuffled_objects_five_objects|3_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:tracking_shuffled_objects_five_objects|3_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index d104bb93d..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:tracking_shuffled_objects_five_objects|3_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1d7e589d611391395b2990a29e55bdd856ab440d45cba22fcd190936daf391dd -size 34842 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:tracking_shuffled_objects_seven_objects|3_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:tracking_shuffled_objects_seven_objects|3_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index dcf7f0c2b..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_bigbench_hard:tracking_shuffled_objects_seven_objects|3_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8e116d939941d57db2e5515114bec4890b56b6a35a5a2e49c809e6361b947337 -size 37387 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_gsm8k_test|0_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_gsm8k_test|0_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index dd2f3049b..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_gsm8k_test|0_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:419f252eddb158e185b515b39ca9e1784f7b4122a620a2a67034178bb1ea6abb -size 35694 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_hellaswag|10_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_hellaswag|10_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index 5730373e4..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_hellaswag|10_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2c74caa34cd1c6b227a1d66dcda7a0c61986435f925f17cf81e676f8c542d146 -size 67250 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_mmlu:college_chemistry|5_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_mmlu:college_chemistry|5_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index 4060e6d9a..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_mmlu:college_chemistry|5_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:25ea151a418829d528da4f762102052308b1cbb15b00d7190d5d0b9fd033436d -size 37684 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_mmlu:us_foreign_policy|5_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_mmlu:us_foreign_policy|5_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index 004081770..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_mmlu:us_foreign_policy|5_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:df6206dba3812d089f03078b97d067eb851eee03fd2aa295cbac2551f82837c0 -size 38453 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_truthfulqa:mc|0_2025-11-05T14-43-47.148527.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_truthfulqa:mc|0_2025-11-05T14-43-47.148527.parquet deleted file mode 100644 index 7cef16523..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-transformers/details_truthfulqa:mc|0_2025-11-05T14-43-47.148527.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d0886aca82f5687e1fd742c0c5b9fe046fb20622d9e67818d21d7300de27e746 -size 26034 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:aqua-rat|0_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:aqua-rat|0_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index e7d35f01f..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:aqua-rat|0_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6337b98efe5a6d10a02d4c13d78bcff056797c65999dbfb8ef5ab128f88fe4cf -size 26482 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:logiqa-en|0_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:logiqa-en|0_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index 190fde6fd..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:logiqa-en|0_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1a998148dbb28f6826861479e8d9fc7bf7f73b0ab6921dc9a6da811e70eddba4 -size 45688 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:lsat-ar|0_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:lsat-ar|0_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index 981c7ea56..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:lsat-ar|0_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:29588411e9390fe550e3ca353e0d7c89e381d25673ced35399f5896e0c613216 -size 50719 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:lsat-lr|0_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:lsat-lr|0_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index 4b1a7c9a1..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:lsat-lr|0_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:450f5a17118613189a749f0fd9f7807265b43733482367e969b04ae7971a749c -size 55774 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:lsat-rc|0_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:lsat-rc|0_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index 26b7aec11..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:lsat-rc|0_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5fab7e1da1cdd0e8f66831b57bcda28b6350c7cf16c9905417746278a6f30f31 -size 148786 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:sat-en-without-passage|0_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:sat-en-without-passage|0_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index 51acd4c8c..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:sat-en-without-passage|0_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:61c1a8942675c523f499e19b01432fb0ce7b0cb8bbd910fe0a16b2b60bb7e80c -size 32704 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:sat-en|0_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:sat-en|0_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index dcc0e80dd..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_agieval:sat-en|0_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:bd4eed73faaf58a18a302a1d2f0c8b8b8e1fbd482a5fd4ca77c375f1e3082f0e -size 109931 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_arc:challenge|25_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_arc:challenge|25_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index c6d8b3b41..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_arc:challenge|25_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7bab826310f526d7aaa9c5e15ff50314524d54847de37699b762adec3c57fb78 -size 144793 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:causal_judgment|3_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:causal_judgment|3_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index e1f456c0d..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:causal_judgment|3_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4b020f10c882cb482d6e2ac597e14d662a5d38f873f5f7eada8c839c5e13b476 -size 72052 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:date_understanding|3_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:date_understanding|3_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index 1ea88c907..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:date_understanding|3_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a8a1f1ef9ba8e3a58d2538f2f2e016769155f2b6c18da49454849c8b276cd398 -size 39423 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:disambiguation_qa|3_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:disambiguation_qa|3_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index a0e5dad68..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:disambiguation_qa|3_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:13fa9f4b64152c7140112b607f6dfddb5f9f755646bbef0b9cc5516a9c0e6de4 -size 38263 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:geometric_shapes|3_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:geometric_shapes|3_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index c928dd875..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:geometric_shapes|3_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5ecdb28b097666cd4ccb0a208708a2076779e06145719ebd24e75060a272bdcf -size 49571 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:logical_deduction_five_objects|3_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:logical_deduction_five_objects|3_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index e1dff3701..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:logical_deduction_five_objects|3_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4582e35192caeae218a50aa76738582d360914fd96cc9a4c3608d3683c44c33a -size 47557 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:logical_deduction_seven_objects|3_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:logical_deduction_seven_objects|3_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index 6c082dfa2..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:logical_deduction_seven_objects|3_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1a4b6292fb5df093df5ac43fba76b0af5b00337e0d2579a9c2b2f6398007b842 -size 56164 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:movie_recommendation|3_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:movie_recommendation|3_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index c80c85e78..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:movie_recommendation|3_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:88e7d66c2396ab8a3f56ae9f4a342037a0f13f4ed83982312fdc7424eb74f60b -size 36502 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:navigate|3_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:navigate|3_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index 2409305c0..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:navigate|3_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e62a25828dddf557091b9dcbc065f2c9e36fdf0c8d365dd1976584fc3f516eae -size 34538 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:ruin_names|3_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:ruin_names|3_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index b8ebb4be1..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:ruin_names|3_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:480bb8570ec01a173ebe85649989dc9a8ab64a4a2de2152d82bc344787bfffee -size 34511 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:salient_translation_error_detection|3_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:salient_translation_error_detection|3_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index 3590ae92e..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:salient_translation_error_detection|3_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ed82ceb8a3c05ae2c47b1769b333173e15069eb83710bc5d66918abb4ef4b7e7 -size 69137 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:snarks|3_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:snarks|3_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index a0e8edf22..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:snarks|3_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:00a0ea645c10c6a8e0d55408f20bf59f95c3cd135afaebc5df0d1fbb89c3b93d -size 37857 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:temporal_sequences|3_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:temporal_sequences|3_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index cdc012856..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:temporal_sequences|3_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:226b2d3fc783dcfecf3c634558746b1314f9f80a32a259c9fe174332fb1e3173 -size 50277 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:tracking_shuffled_objects_five_objects|3_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:tracking_shuffled_objects_five_objects|3_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index dad411769..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:tracking_shuffled_objects_five_objects|3_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f6dd8f8d104f1a4252685019e5413ce9ecfc4611bb819ff627e77be296afc581 -size 52493 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:tracking_shuffled_objects_seven_objects|3_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:tracking_shuffled_objects_seven_objects|3_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index 38b48b536..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_bigbench_hard:tracking_shuffled_objects_seven_objects|3_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9651280724d245b37a7c3dde465c5a384de7b12055b9474696d533d58330b240 -size 59838 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_gsm8k_test|0_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_gsm8k_test|0_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index dfebfe142..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_gsm8k_test|0_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:df34c40c43eeea4355e86ec505b053db421189b2082c670409e66d93defdd0d1 -size 39054 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_hellaswag|10_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_hellaswag|10_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index b54d084d0..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_hellaswag|10_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:973fa1740490bf212831075ac9842dd88a31db7aa422e240c01eafb840979207 -size 88599 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_mmlu:college_chemistry|5_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_mmlu:college_chemistry|5_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index e08f92d91..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_mmlu:college_chemistry|5_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:fa71775ee3b585b1612f2bbbd8327ba4f564c9eddcdce63533fd6d11c67c5d95 -size 50977 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_mmlu:us_foreign_policy|5_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_mmlu:us_foreign_policy|5_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index 68d60ab99..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_mmlu:us_foreign_policy|5_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c5cae6c52cee4a49e9cf28920ec303235a15f2b9f075b4f2deff65bd220aea77 -size 52510 diff --git a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_truthfulqa:mc|0_2025-11-05T14-52-08.352779.parquet b/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_truthfulqa:mc|0_2025-11-05T14-52-08.352779.parquet deleted file mode 100644 index 871401f6b..000000000 --- a/tests/reference_details/SmolLM2-1.7B-Instruct-vllm/details_truthfulqa:mc|0_2025-11-05T14-52-08.352779.parquet +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ec46468169068183da1c57ace7064fcfa8664e4acad3a76f2d37e260468b67ee -size 32367 diff --git a/tests/reference_scores/Qwen2.5-VL-3B-Instruct-results-vlm.json b/tests/reference_scores/Qwen2.5-VL-3B-Instruct-results-vlm.json deleted file mode 100644 index d5e18b444..000000000 --- a/tests/reference_scores/Qwen2.5-VL-3B-Instruct-results-vlm.json +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5b38f65703ddd426111ba45e6f6b8b82ee2049c7e754e977a5c6269aa2d94ade -size 3968 diff --git a/tests/reference_scores/Qwen2.5-VL-7B-Instruct-results-vlm.json b/tests/reference_scores/Qwen2.5-VL-7B-Instruct-results-vlm.json deleted file mode 100644 index b9080f9f7..000000000 --- a/tests/reference_scores/Qwen2.5-VL-7B-Instruct-results-vlm.json +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:d07d8341188999f359a530e1dae4cd8ec3936d4046232a68b90a56c9f2994b3c -size 3083 diff --git a/tests/reference_scores/SmolLM2-1.7B-Instruct-results-accelerate.json b/tests/reference_scores/SmolLM2-1.7B-Instruct-results-accelerate.json deleted file mode 100644 index af7972912..000000000 --- a/tests/reference_scores/SmolLM2-1.7B-Instruct-results-accelerate.json +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:55b420e5ff6061d2d2d66c3f9ce8cab541820766b1bd7becc0d7b290b99144b6 -size 47858 diff --git a/tests/reference_scores/SmolLM2-1.7B-Instruct-results-vllm.json b/tests/reference_scores/SmolLM2-1.7B-Instruct-results-vllm.json deleted file mode 100644 index a63acbfdb..000000000 --- a/tests/reference_scores/SmolLM2-1.7B-Instruct-results-vllm.json +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4dcc899c5963df3e98cc9d144f3c582edda227d8d9e2c24fabc1f794a4fab524 -size 47986 diff --git a/tests/reference_scores/harness_metrics.json b/tests/reference_scores/harness_metrics.json deleted file mode 100644 index 8f4be7807..000000000 --- a/tests/reference_scores/harness_metrics.json +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:c2080305011a7ac8b0895ec1fbb26b45af4e3dced6272abf67156ebf57656f88 -size 48360080 diff --git a/tests/reference_scores/harness_prompts.json b/tests/reference_scores/harness_prompts.json deleted file mode 100644 index 9d42913f3..000000000 --- a/tests/reference_scores/harness_prompts.json +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:059a48631d4243cda36d067db50350639c12b0a88fb209f76bbcd0c3ff266ffb -size 20244711 diff --git a/tests/slow_tests/__init__.py b/tests/slow_tests/__init__.py deleted file mode 100644 index a732db8d0..000000000 --- a/tests/slow_tests/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. diff --git a/tests/slow_tests/sample_comparison.py b/tests/slow_tests/sample_comparison.py deleted file mode 100644 index 393d8943f..000000000 --- a/tests/slow_tests/sample_comparison.py +++ /dev/null @@ -1,253 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import math -from dataclasses import asdict -from pathlib import Path - -from datasets import Dataset - - -def _to_plain_list(value): - """convert a list of tensors to a list of plain values""" - new_value = [] - for item in value: - if hasattr(item, "tolist"): - item = item.tolist() - new_value.append(item) - return new_value - - -def load_sample_details(details_dir: str): - """Load sample-level details from parquet files in the details directory.""" - details = {} - details_path = Path(details_dir) - - if not details_path.exists(): - return details - - for parquet_file in details_path.glob("details_*.parquet"): - # Extract task name from parquet filename, keeping the full task path with "|" separators - task_name = parquet_file.stem.replace("details_", "").rsplit("_", 1)[ - 0 - ] # Split from right to preserve task name with "|" - dataset = Dataset.from_parquet(str(parquet_file)) - details[task_name] = list(dataset) - - return details - - -def _compare_model_responses(current, reference): - """Compare model response fields between current and reference.""" - sample_diff = {} - current_resp = current["model_response"] - reference_resp = reference["model_response"] - - # Get all field names from both responses - all_fields = set(current_resp.keys()) | set(reference_resp.keys()) - - for field_name in all_fields: - current_val = current_resp.get(field_name) - reference_val = reference_resp.get(field_name) - - # Special handling for logprobs field - if field_name in ["input_tokens", "output_tokens"]: - # input and ouput tokens are lists of tensors, we need to convert - # them to plain lists - current_val = _to_plain_list(current_val) - reference_val = _to_plain_list(reference_val) - - if current_val != reference_val: - sample_diff["{}_difference".format(field_name)] = { - "current": current_val, - "reference": reference_val, - } - - return sample_diff - - -def _compare_metrics(current, reference): - """Compare metric fields between current and reference.""" - sample_diff = {} - current_metrics = current["metric"] - reference_metrics = reference["metric"] - - metric_diffs = {} - for metric_name in set(current_metrics.keys()) & set(reference_metrics.keys()): - current_val = current_metrics.get(metric_name) - reference_val = reference_metrics.get(metric_name) - - if not math.isclose(current_val, reference_val, abs_tol=0.05): - metric_diffs[metric_name] = {"current": current_val, "reference": reference_val} - - if metric_diffs: - sample_diff["metric_differences"] = metric_diffs - - return sample_diff - - -def _compare_doc_info(current, reference): - """Compare document information between current and reference.""" - sample_diff = {} - current_doc = current["doc"] - reference_doc = reference["doc"] - - if current_doc.get("query") != reference_doc.get("query"): - sample_diff["query_difference"] = { - "current": current_doc.get("query"), - "reference": reference_doc.get("query"), - } - - if current_doc.get("choices") != reference_doc.get("choices"): - sample_diff["choices_difference"] = { - "current": current_doc.get("choices"), - "reference": reference_doc.get("choices"), - } - - return sample_diff - - -def _compare_single_sample(current, reference, sample_index): - """Compare a single sample between current and reference.""" - sample_diff = {} - current = asdict(current) - - if "model_response" in current and "model_response" in reference: - sample_diff.update(_compare_model_responses(current, reference)) - - if "metric" in current and "metric" in reference: - sample_diff.update(_compare_metrics(current, reference)) - - if "doc" in current and "doc" in reference: - sample_diff.update(_compare_doc_info(current, reference)) - - if sample_diff: - sample_diff["sample_index"] = sample_index - - return sample_diff - - -def compare_sample_details(current_details, reference_details): - """Compare sample-by-sample details between current and reference results.""" - differences = {} - - for task_name in current_details: - if task_name not in reference_details: - differences[task_name] = [{"error": "Task not found in reference results"}] - continue - - current_samples = current_details[task_name] - reference_samples = reference_details[task_name] - - if len(current_samples) != len(reference_samples): - differences[task_name] = [ - {"error": f"Sample count mismatch: current={len(current_samples)}, reference={len(reference_samples)}"} - ] - continue - - task_differences = [] - for i, (current, reference) in enumerate(zip(current_samples, reference_samples)): - sample_diff = _compare_single_sample(current, reference, i) - if sample_diff: - task_differences.append(sample_diff) - - if task_differences: - differences[task_name] = task_differences - - return differences - - -def _format_single_diff(diff): - """Format a single sample difference.""" - output = [] - sample_idx = diff.get("sample_index", "unknown") - output.append(" Sample {}:".format(sample_idx)) - - # Handle model response field differences - for key, value in diff.items(): - if key.endswith("_difference") and key != "metric_differences": - field_name = key.replace("_difference", "") - output.append(" {} differs:".format(field_name.title())) - output.append(" Current: {}".format(value["current"])) - output.append(" Reference: {}".format(value["reference"])) - - # Handle metric differences - if "metric_differences" in diff: - output.append(" Metrics differ:") - for metric_name, metric_diff in diff["metric_differences"].items(): - output.append( - " {}: current={}, reference={}".format( - metric_name, metric_diff["current"], metric_diff["reference"] - ) - ) - - return output - - -def format_sample_differences(differences): - """Format sample differences into a readable string for test output.""" - if not differences: - return "No sample-level differences found." - - output = ["Sample-by-sample differences found:"] - - for task_name, task_diffs in differences.items(): - output.append(f"\nTask: {task_name}") - - for diff in task_diffs: - if "error" in diff: - output.append(f" ERROR: {diff['error']}") - continue - - output.extend(_format_single_diff(diff)) - - return "\n".join(output) - - -def enhance_test_with_sample_comparison(diff, details, reference_details_dir): - """Enhance test failure messages with sample-by-sample comparison details.""" - # Load reference details if available - reference_details = load_sample_details(reference_details_dir) - - # Compare sample-by-sample details - sample_differences = compare_sample_details(details, reference_details) - - # Always check for sample-level differences, even if high-level results match - if sample_differences: - sample_diff_message = format_sample_differences(sample_differences) - - if diff != {}: - # Both high-level and sample-level differences found - full_message = f"High-level differences found: {diff}\n\n{sample_diff_message}" - else: - # Only sample-level differences found (high-level results match) - full_message = f"High-level results match, but sample-level differences found:\n\n{sample_diff_message}" - - return full_message - - # No sample-level differences found - if diff != {}: - # Only high-level differences found - return f"High-level differences found: {diff}" - - # No differences at any level - return None diff --git a/tests/slow_tests/test_accelerate_model.py b/tests/slow_tests/test_accelerate_model.py deleted file mode 100644 index fdf6104cf..000000000 --- a/tests/slow_tests/test_accelerate_model.py +++ /dev/null @@ -1,111 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import json -import os -from functools import lru_cache, partial -from typing import Callable, Tuple - -import pytest -from deepdiff import DeepDiff - -from lighteval.main_accelerate import accelerate # noqa: E402 -from tests.slow_tests.sample_comparison import enhance_test_with_sample_comparison - - -# Set env var for deterministic run of models -os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" - -MODELS_ARGS = [ - { - "model_name": "examples/model_configs/transformers_model.yaml", - "results_file": "tests/reference_scores/SmolLM2-1.7B-Instruct-results-accelerate.json", - } -] -TASKS_PATH = "examples/test_tasks.txt" -CUSTOM_TASKS_PATH = "examples/custom_tasks_tests.py" -DETAILS_EXPECTED_DIR = "tests/reference_details/SmolLM2-1.7B-Instruct-transformers/" - -ModelInput = Tuple[str, Callable[[], dict]] - - -@lru_cache(maxsize=len(MODELS_ARGS)) -def run_model(model_name: str): - """Runs the full main as a black box, using the input model and tasks, on 10 samples without parallelism""" - results, details = accelerate( - model_args=model_name, - tasks=TASKS_PATH, - output_dir="", - dataset_loading_processes=1, - save_details=False, - max_samples=10, - custom_tasks=CUSTOM_TASKS_PATH, - ) - return results, details - - -def generate_tests() -> list[ModelInput]: - """Generate test parameters for all models and tasks.""" - - tests = [] - for model_args in MODELS_ARGS: - predictions_lite = partial(run_model, model_args["model_name"]) - tests.append((model_args, predictions_lite)) - - return tests - - -# generates the model predictions parameters at test collection time -tests: list[ModelInput] = generate_tests() -ids = [f"{model_input[0]['model_name']}" for model_input in tests] - - -@pytest.mark.slow -@pytest.mark.parametrize("tests", tests, ids=ids) -def test_accelerate_model_prediction(tests: list[ModelInput]): - """Evaluates a model on a full task - is parametrized using pytest_generate_test""" - model_args, get_predictions = tests - - # Load the reference results - with open(model_args["results_file"], "r") as f: - reference_results = json.load(f)["results"] - - # Change the key names, replace '|' with ':' - reference_results = {k.replace("|", ":"): v for k, v in reference_results.items()} - - # Get the predictions - results, details = get_predictions() - results = results["results"] - - # Convert defaultdict values to regular dict for comparison - predictions_dict = {k: dict(v) if hasattr(v, "default_factory") else v for k, v in results.items()} - - # Compare the predictions with the reference results - diff = DeepDiff(reference_results, predictions_dict, ignore_numeric_type_changes=True, math_epsilon=0.05) - - # Always check for sample-by-sample differences, even if high-level results match - enhanced_message = enhance_test_with_sample_comparison(diff, details, DETAILS_EXPECTED_DIR) - if enhanced_message: - assert False, enhanced_message - else: - # No differences found at any level, test passes - assert True diff --git a/tests/slow_tests/test_accelerate_vlm_model.py b/tests/slow_tests/test_accelerate_vlm_model.py deleted file mode 100644 index df611364a..000000000 --- a/tests/slow_tests/test_accelerate_vlm_model.py +++ /dev/null @@ -1,110 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import json -import os -from functools import lru_cache, partial -from typing import Callable, Tuple - -import pytest -from deepdiff import DeepDiff - -from lighteval.main_accelerate import accelerate # noqa: E402 -from tests.slow_tests.sample_comparison import enhance_test_with_sample_comparison - - -# Set env var for deterministic run of models -os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" - -MODELS_ARGS = [ - { - "model_name": "examples/model_configs/transformers_vlm_model.yaml", - "results_file": "tests/reference_scores/Qwen2.5-VL-3B-Instruct-results-vlm.json", - } -] -TASKS = "mmmu_pro:standard-4" -DETAILS_EXPECTED_DIR = "tests/reference_details/Qwen2.5-VL-3B-Instruct-vlm/" - -ModelInput = Tuple[str, Callable[[], dict]] - - -@lru_cache(maxsize=len(MODELS_ARGS)) -def run_model(model_name: str): - """Runs the full main as a black box, using the input model and tasks, on 10 samples without parallelism""" - results, details = accelerate( - model_args=model_name, - tasks=TASKS, - output_dir="", - dataset_loading_processes=1, - save_details=False, - max_samples=30, - vision_model=True, - ) - return results, details - - -def generate_tests() -> list[ModelInput]: - """Generate test parameters for all models and tasks.""" - - tests = [] - for model_args in MODELS_ARGS: - predictions_lite = partial(run_model, model_args["model_name"]) - tests.append((model_args, predictions_lite)) - - return tests - - -# generates the model predictions parameters at test collection time -tests: list[ModelInput] = generate_tests() -ids = [f"{model_input[0]['model_name']}" for model_input in tests] - - -@pytest.mark.slow -@pytest.mark.parametrize("tests", tests, ids=ids) -def test_accelerate_model_prediction(tests: list[ModelInput]): - """Evaluates a model on a full task - is parametrized using pytest_generate_test""" - model_args, get_predictions = tests - - # Load the reference results - with open(model_args["results_file"], "r") as f: - reference_results = json.load(f)["results"] - - # Change the key names, replace '|' with ':' - reference_results = {k.replace("|", ":"): v for k, v in reference_results.items()} - - # Get the predictions - results, details = get_predictions() - results = results["results"] - - # Convert defaultdict values to regular dict for comparison - predictions_dict = {k: dict(v) if hasattr(v, "default_factory") else v for k, v in results.items()} - - # Compare the predictions with the reference results - diff = DeepDiff(reference_results, predictions_dict, ignore_numeric_type_changes=True, math_epsilon=0.05) - - # Always check for sample-by-sample differences, even if high-level results match - enhanced_message = enhance_test_with_sample_comparison(diff, details, DETAILS_EXPECTED_DIR) - if enhanced_message: - assert False, enhanced_message - else: - # No differences found at any level, test passes - assert True diff --git a/tests/slow_tests/test_vllm_model.py b/tests/slow_tests/test_vllm_model.py deleted file mode 100644 index def25dedb..000000000 --- a/tests/slow_tests/test_vllm_model.py +++ /dev/null @@ -1,108 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import json -import os -from functools import lru_cache, partial -from typing import Callable, Tuple - -import pytest -from deepdiff import DeepDiff - -from lighteval.main_vllm import vllm # noqa: E402 -from tests.slow_tests.sample_comparison import enhance_test_with_sample_comparison - - -# Set env var for deterministic run of models -os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" - -MODELS_ARGS = [ - { - "model_name": "examples/model_configs/vllm_model_config.yaml", - "results_file": "tests/reference_scores/SmolLM2-1.7B-Instruct-results-vllm.json", - } -] -TASKS_PATH = "examples/test_tasks.txt" -CUSTOM_TASKS_PATH = "examples/custom_tasks_tests.py" -DETAILS_EXPECTED_DIR = "tests/reference_details/SmolLM2-1.7B-Instruct-vllm/" - -ModelInput = Tuple[str, Callable[[], dict]] - - -@lru_cache(maxsize=len(MODELS_ARGS)) -def run_model(model_name: str): - """Runs the full main as a black box, using the input model and tasks, on 10 samples without parallelism""" - results = vllm( - model_args=model_name, - tasks=TASKS_PATH, - output_dir="", - dataset_loading_processes=1, - save_details=False, - max_samples=10, - custom_tasks=CUSTOM_TASKS_PATH, - ) - return results - - -def generate_tests() -> list[ModelInput]: - """Generate test parameters for all models and tasks.""" - - tests = [] - for model_args in MODELS_ARGS: - predictions_lite = partial(run_model, model_args["model_name"]) - tests.append((model_args, predictions_lite)) - - return tests - - -# generates the model predictions parameters at test collection time -tests: list[ModelInput] = generate_tests() -ids = [f"{model_input[0]['model_name']}" for model_input in tests] - - -@pytest.mark.slow -@pytest.mark.parametrize("tests", tests, ids=ids) -def test_vllm_model(tests: list[ModelInput]): - """Evaluates a model on a full task - is parametrized using pytest_generate_test""" - model_args, get_predictions = tests - - results, details = get_predictions() - results = results["results"] - - # Load the reference results - with open(model_args["results_file"], "r") as f: - reference_results = json.load(f)["results"] - - # Change the key names, replace '|' with ':' - reference_results = {k.replace("|", ":"): v for k, v in reference_results.items()} - - # Convert defaultdict values to regular dict for comparison - predictions_dict = {k: dict(v) if hasattr(v, "default_factory") else v for k, v in results.items()} - - diff = DeepDiff(reference_results, predictions_dict, ignore_numeric_type_changes=True, math_epsilon=0.05) - - enhanced_message = enhance_test_with_sample_comparison(diff, details, DETAILS_EXPECTED_DIR) - if enhanced_message: - assert False, enhanced_message - else: - # No differences found at any level, test passes - assert True diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py deleted file mode 100644 index 694e5ec47..000000000 --- a/tests/test_dependencies.py +++ /dev/null @@ -1,79 +0,0 @@ -# MIT License -# -# Copyright (c) 2024 The HuggingFace Team -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import pytest - -from lighteval.utils.imports import Extra, is_package_available, requires - - -def test_requires(): - @requires("sglang") - class RandomModel: - pass - - assert RandomModel.__name__ == "RandomModel" - assert RandomModel.__class__.__name__ == "DummyObject" - - with pytest.raises( - ImportError, - match="Through the use of RandomModel, you requested the use of `sglang` for this evaluation, but it is not available in your current environment. Please install it using pip.", - ): - RandomModel() - - -def test_requires_with_extra(): - @requires(Extra.TGI) - class RandomModel: - pass - - with pytest.raises( - ImportError, - match=r"Through the use of RandomModel, you are trying to run an evaluation requiring tgi capabilities. Please install the required extra: `pip install lighteval\[tgi\]`", - ): - RandomModel() - - -def test_requires_with_wrong_dependency(): - with pytest.raises( - RuntimeError, - match="A dependency was specified with @requires, but it is not defined in the possible dependencies defined in the pyproject.toml: `random_dependency`", - ): - - @requires("random_dependency") - class RandomModel: - pass - - -def test_is_package_available(): - assert is_package_available("torch") - - -def test_is_package_unavailable(): - assert not is_package_available("tensorboardX") - - -def test_is_package_is_not_specified_in_pyproject_toml(): - with pytest.raises( - RuntimeError, - match="Package tensorflow was tested against, but isn't specified in the pyproject.toml file. Please specifyit as a potential dependency or an extra for it to be checked.", - ): - is_package_available("tensorflow") diff --git a/tests/test_unit_base_metrics.py b/tests/test_unit_base_metrics.py deleted file mode 100644 index 575ebf595..000000000 --- a/tests/test_unit_base_metrics.py +++ /dev/null @@ -1,340 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import numpy as np -import pytest - -from lighteval.metrics.dynamic_metrics import ( - LogLikelihoodAccMetric, - MultilingualQuasiExactMatchMetric, - MultilingualQuasiF1ScoreMetric, - NormalizedMultiChoiceProbMetric, - ProbabilityMetric, -) -from lighteval.metrics.metrics_sample import ExactMatches -from lighteval.metrics.normalizations import LogProbCharNorm, helm_normalizer -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.requests import Doc -from lighteval.utils.language import Language - - -class TestBaseMetrics: - def test_exact_match(self): - em = ExactMatches(strip_strings=True) - - res = em.compute_one_item( - "The quick brown fox jumps over the lazy dog", - "The quick brown fox jumps over the lazy dog", - ) - assert res == 1 - - res = em.compute_one_item( - " The quick brown fox jumps over the lazy dog\n", - "\n The quick brown fox jumps over the lazy dog ", - ) - assert res == 1 - - res = em.compute_one_item( - "The quick brown fox jumps over the lazy dog", - "The quick brown fox jumps over the lazy dog.", - ) - assert res == 0 - - res = em.compute_one_item("The quick brown fox jumps over the lazy dog", "") - assert res == 0 - - res = em.compute_one_item("", "") - assert res == 0 - - def test_quasi_exact_match(self): - em = ExactMatches(normalize_gold=helm_normalizer, normalize_pred=helm_normalizer) - - res = em.compute_one_item( - "The quick brown fox jumps over the lazy dog", - "The quick brown fox jumps over the lazy dog", - ) - assert res == 1 - - res = em.compute_one_item( - " The quick brown fox jumps over the lazy dog\n", - "\n The quick brown fox jumps over the lazy dog ", - ) - assert res == 1 - - res = em.compute_one_item( - "The quick brown fox jumps over the lazy dog", - "The quick brown fox jumps over the lazy dog.", - ) - assert res == 1 - - res = em.compute_one_item("the quick brown fox, jumps over lazy dog", "quick brown fox jumps over lazy dog.") - assert res == 1 - - res = em.compute_one_item("The quick brown fox jumps over the lazy dog", "") - assert res == 0 - - res = em.compute_one_item("", "") - assert res == 0 - - def test_prefix_exact_match(self): - em = ExactMatches( - strip_strings=True, - type_exact_match="prefix", - ) - - res = em.compute_one_item( - "The quick brown fox jumps over the lazy dog", - "The quick brown fox jumps over the lazy dog", - ) - assert res == 1 - - res = em.compute_one_item( - "The quick brown fox jumps over the lazy dog", - "The quick brown fox jumps over the lazy dog. And some other stories.", - ) - assert res == 1 - - res = em.compute_one_item( - " The quick brown fox jumps over the lazy dog\n", - "\n The quick brown fox jumps over the lazy dog", - ) - assert res == 1 - - res = em.compute_one_item( - "The quick brown fox jumps over the lazy dog", - "The quick brown fox jumps over the lazy dog.", - ) - assert res == 1 - - res = em.compute_one_item( - "The quick brown fox jumps over the lazy dog", - "the quick brown fox jumps over lazy dog. And some other stories.", - ) - assert res == 0 - - res = em.compute_one_item("The quick brown fox jumps over the lazy dog", "") - assert res == 0 - - res = em.compute_one_item( - "The quick brown fox jumps over the lazy dog", - "Complete mismatch", - ) - assert res == 0 - - res = em.compute_one_item("", "") - assert res == 0 - - def test_prefix_quasi_exact_match(self): - em = ExactMatches( - normalize_gold=helm_normalizer, - normalize_pred=helm_normalizer, - type_exact_match="prefix", - ) - res = em.compute_one_item( - "The quick brown fox jumps over the lazy dog", - "The quick brown fox jumps over the lazy dog", - ) - assert res == 1 - - res = em.compute_one_item( - "The quick brown fox jumps over the lazy dog", - "The quick brown fox jumps over the lazy dog. And some other stories.", - ) - assert res == 1 - - res = em.compute_one_item( - "The quick Brown fox jumps over the lazy dog", - "the quick brown fox jumps over lazy dog. And some other stories.", - ) - assert res == 1 - - res = em.compute_one_item( - " The quick brown fox jumps over the lazy dog\n", - "\n The quick brown fox jumps over the lazy dog", - ) - assert res == 1 - - res = em.compute_one_item( - "The quick brown fox jumps over the lazy dog", - "The quick brown fox jumps over the lazy dog.", - ) - assert res == 1 - - res = em.compute_one_item("The quick brown fox jumps over the lazy dog", "") - assert res == 0 - - res = em.compute_one_item( - "The quick brown fox jumps over the lazy dog", - "Complete mismatch", - ) - assert res == 0 - - res = em.compute_one_item("", "") - assert res == 0 - - def test_prob(self): - doc = Doc(query="Test query", choices=["A", "B", "C"], gold_index=0, task_name="test") - - # Simple case - model_response = ModelResponse(logprobs=np.log([0.7])) - prob_metric = ProbabilityMetric() - result = prob_metric.compute_sample(doc=doc, model_response=model_response) - assert result[prob_metric.metric_name] == pytest.approx(0.7) - - # Aggregation function test - model_response = ModelResponse(logprobs=np.log([0.7, 0.1])) - prob_min_metric = ProbabilityMetric(aggregation_function=np.min) - result = prob_min_metric.compute_sample(doc=doc, model_response=model_response) - assert result[prob_metric.metric_name] == pytest.approx(0.1) - - def test_mc_probability_metric(self): - doc = Doc(query="Test query", choices=["A", "B", "C"], gold_index=0, task_name="test") - model_response = ModelResponse(logprobs=np.log([0.35, 0.1, 0.05])) - - mc_prob_metric = NormalizedMultiChoiceProbMetric() - - result = mc_prob_metric.compute_sample( - doc=doc, - model_response=model_response, - ) - assert result[mc_prob_metric.metric_name] == pytest.approx(0.7) - - doc = Doc(query="Test query", choices=["AA", "BB", "CCC"], gold_index=1, task_name="test") - model_response = ModelResponse(logprobs=np.log([0.1**2, 0.35**2, 0.05**3])) - - prob_norm_metric = NormalizedMultiChoiceProbMetric(normalization=LogProbCharNorm()) - result = prob_norm_metric.compute_sample( - doc=doc, - model_response=model_response, - ) - assert result[prob_norm_metric.metric_name] == pytest.approx(0.7) - - def test_acc(self): - # Test without normalization - doc = Doc(query="Test query", choices=["A", "B", "C", "D"], gold_index=0, task_name="test") - model_response = ModelResponse(logprobs=np.log([0.7, 0.2, 0.3, 0.4])) - - acc_metric = LogLikelihoodAccMetric() - result = acc_metric.compute_sample( - doc=doc, - model_response=model_response, - ) - assert result[acc_metric.metric_name] == 1 # The highest logprob (3.0) is at index 3, which is not in gold_ixs - - # Test 0 acc - doc = Doc(query="Test query", choices=["A", "B", "C", "D"], gold_index=0, task_name="test") - model_response = ModelResponse(logprobs=np.log([0.1, 0.2, 0.3, 0.4])) - result = acc_metric.compute_sample( - doc=doc, - model_response=model_response, - ) - assert result[acc_metric.metric_name] == 0 - - # Test with normalization - doc = Doc(query="Test query", choices=["ABCDE", "AB"], gold_index=0, task_name="test") - model_response = ModelResponse(logprobs=np.log([0.5, 0.6])) - acc_norm_metric = LogLikelihoodAccMetric(normalization=LogProbCharNorm()) - result_norm = acc_norm_metric.compute_sample( - doc=doc, - model_response=model_response, - ) - assert ( - result_norm[acc_norm_metric.metric_name] == 1 - ) # After normalization, "ABCDE" should have the highest score - - # Test with multiple correct solutions - doc = Doc(query="Test query", choices=["A", "B", "C", "D"], gold_index=[1, 3], task_name="test") - model_response = ModelResponse(logprobs=np.log([0.5, 0.6, 0.7, 0.8])) - result_multi = acc_metric.compute_sample( - doc=doc, - model_response=model_response, - ) - assert result_multi[acc_metric.metric_name] == 1 - - # Test when the highest logprob is not in gold_ixs - doc = Doc(query="Test query", choices=["A", "B", "C", "D"], gold_index=[1, 2], task_name="test") - model_response = ModelResponse(logprobs=[0.5, 0.6, 0.7, 0.8]) - result_incorrect = acc_metric.compute_sample( - doc=doc, - model_response=model_response, - ) - assert result_incorrect[acc_metric.metric_name] == 0 - - def test_f1_dynamic_metric(self): - """ - Tests that normalization works correctly. We don't test the behavior of the F1_score class as it should be already tested. - """ - - doc = Doc(query="Test query", choices=["hello world"], gold_index=[0], task_name="test") - model_response = ModelResponse(text=["hello, the world"]) - - # Normalization test - f1_metric = MultilingualQuasiF1ScoreMetric(language=Language.ENGLISH) - result = f1_metric.compute_sample( - doc=doc, - model_response=model_response, - ) - assert result[f1_metric.metric_name] == 1 - - model_response = ModelResponse(text=["hello, the world how"]) - f1_metric = MultilingualQuasiF1ScoreMetric(language=Language.ENGLISH, aggregation_function=np.min) - result = f1_metric.compute_sample( - doc=doc, - model_response=model_response, - ) - # 2 * (precision * recall) / (precision + recall) = 2 * (1 * 2/3) / (1 + 2/3) = 0.8 - assert result[f1_metric.metric_name] == 0.8 - - def test_exact_match_dynamic_metric(self): - """ - Tests that normalization works correctly. We don't test the behavior of the ExactMatch class as it should be already tested. - """ - doc = Doc(query="Test query", choices=["hello world"], gold_index=[0], task_name="test") - model_response = ModelResponse(text=["hello, the world"]) - - # Normalization test - em_metric = MultilingualQuasiExactMatchMetric(language=Language.ENGLISH, match_type="full") - result = em_metric.compute_sample( - doc=doc, - model_response=model_response, - ) - assert result[em_metric.metric_name] == 1 - - model_response = ModelResponse(text=["hello, the world how"]) - em_metric = MultilingualQuasiExactMatchMetric(language=Language.ENGLISH, match_type="full") - result = em_metric.compute_sample( - doc=doc, - model_response=model_response, - ) - assert result[em_metric.metric_name] == 0 - - @pytest.mark.skip(reason="Need to understand what it does.") - def test_pass_at_k_estimator(self): - assert False - - @pytest.mark.skip(reason="Using nltk metric function, no need to test.") - def test_f1_score_quasi(self): - assert False - - @pytest.mark.skip(reason="Using nltk metric function, no need to test.") - def test_f1(self): - assert False diff --git a/tests/test_unit_harness_metrics.py b/tests/test_unit_harness_metrics.py deleted file mode 100644 index 6d1764593..000000000 --- a/tests/test_unit_harness_metrics.py +++ /dev/null @@ -1,139 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import json -import os - -import pytest - -from lighteval.metrics import apply_metric -from lighteval.metrics.metrics import Metrics -from lighteval.metrics.sample_preparator import ( - GenerativeCorpusMetricInput, - LogprobCorpusMetricInput, - PerplexityCorpusMetricInput, -) -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.requests import Doc -from lighteval.utils.utils import as_list - - -PATH_TO_HARNESS_METRICS = os.path.join(os.path.dirname(__file__), "reference_scores/harness_metrics.json") - - -def pytest_generate_tests(metafunc: pytest.Metafunc): - """Initializes the main test setup. This function is automatically called by pytest and - should not be called manually. - - Every function with "model_input" as arguments will be sent the "parameters". - This function will be run only once, ensuring that each model is run only once on the selected tasks. - (This is better than using fixtures as fixtures are re-run once for each test, which is not a behavior we want). - """ - parameters = [] - - # If model_input is a test function argument - # (= the function requires a fixture) - if "prompt_inputs" in metafunc.fixturenames: - with open(PATH_TO_HARNESS_METRICS) as f: - metric_to_examples = json.load(f) - - for metric, examples in metric_to_examples.items(): - for task_name, examples_list in examples.items(): - parameters.append((metric, task_name, examples_list)) - metafunc.parametrize("prompt_inputs", parameters, scope="session") - - -def test_model_prediction(prompt_inputs: tuple[str, str, list]): # noqa: C901 - """Evaluates a model on a full task - is parametrized using pytest_generate_test""" - metric, task_name, examples = prompt_inputs - metric_name = metric - metric = Metrics[metric].value - - for example in examples: - doc = { - k: v - for k, v in example.items() - if k in ["full_prompt", "choices", "gold_index", "original_query", "specific"] - } - doc["query"] = doc.pop("full_prompt") - doc = Doc(**doc) - error_msg = f"Metric {metric_name} failed on input {doc} from task {task_name}.\n" - - match example["predictions"]: - case [first_element, *_] if isinstance(first_element, str): - # If the predictions are a list of strings, we assume it's a generative task - responses = [ModelResponse(text=example["predictions"], output_tokens=[[]], input_tokens=[])] - case [first_element, *_] if isinstance(first_element, float): - # If the predictions are a list of floats, we assume it's a logprob task - responses = [ModelResponse(logprobs=example["predictions"], output_tokens=[[]], input_tokens=[])] - case [first_element, *_] if len(first_element) == 2 and isinstance(first_element[1], bool): - # If the predictions are a list of lists with two elements, we assume it's a loglikelihood task with argmax - responses = [ - ModelResponse( - logprobs=[pred[0] for pred in example["predictions"]], - argmax_logits_eq_gold=[pred[1] for pred in example["predictions"]], - output_tokens=[[]], - input_tokens=[], - ) - ] - case _: - # If the predictions are not a list of strings or floats, we assume it's a custom task - responses = [ModelResponse(logprobs=example["predictions"][0], input_tokens=[])] - - results = apply_metric(responses=responses, docs=[doc], metrics=[metric])[0] - assert responses is not None, error_msg - - metric_result = {k: list(v) if isinstance(v, tuple) else v for k, v in results.items()} - - metric_reference = {k: example[k] for k in results.keys()} - error_msg += f"Prediction: {results}\n" - error_msg += f"Reference: {metric_reference}\n" - error_msg += f"Returned : {metric_result}" - - for key in metric_result.keys(): - if type(metric_result[key]) in [ - LogprobCorpusMetricInput, - GenerativeCorpusMetricInput, - PerplexityCorpusMetricInput, - ]: - cur_result_list = as_list(metric_result[key].to_dict()) - else: - cur_result_list = as_list(metric_result[key]) - cur_ref_list = as_list(metric_reference[key]) - - # item wise comparison of lists - if isinstance(cur_result_list[0], list): - for res, ref in zip(cur_result_list, cur_ref_list): - try: - assert res == pytest.approx(ref, rel=1e-8), error_msg - except Exception: - assert False, ( - key + "\n" + str(cur_result_list) + "\n" + str(cur_ref_list) + "\n" + task_name + "\n" - ) - else: - try: - assert cur_result_list == pytest.approx(cur_ref_list, rel=1e-8), error_msg - except Exception: - # assert False, error_msg + "\n" + str(e) - assert False, ( - key + "\n" + str(cur_result_list) + "\n" + str(cur_ref_list) + "\n" + task_name + "\n" - ) diff --git a/tests/unit/logging/test_evaluation_tracker.py b/tests/unit/logging/test_evaluation_tracker.py deleted file mode 100644 index 45c5790d0..000000000 --- a/tests/unit/logging/test_evaluation_tracker.py +++ /dev/null @@ -1,487 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import json -import os -import tempfile -from datetime import datetime -from pathlib import Path - -import pytest -from datasets import Dataset -from huggingface_hub import HfApi - -from lighteval.logging.evaluation_tracker import EvaluationTracker -from lighteval.logging.info_loggers import DetailsLogger - -# ruff: noqa -from tests.fixtures import TESTING_EMPTY_HF_ORG_ID -from unittest.mock import patch, Mock -import unittest - - -@pytest.fixture -def mock_evaluation_tracker(request): - passed_params = {} - if request.keywords.get("evaluation_tracker"): - passed_params = request.keywords["evaluation_tracker"].kwargs - - with tempfile.TemporaryDirectory() as temp_dir: - kwargs = { - "output_dir": temp_dir, - "save_details": passed_params.get("save_details", False), - "push_to_hub": passed_params.get("push_to_hub", False), - "push_to_tensorboard": passed_params.get("push_to_tensorboard", False), - "hub_results_org": passed_params.get("hub_results_org", ""), - } - tracker = EvaluationTracker(**kwargs) - tracker.general_config_logger.model_name = "test_model" - - # Create a dummy model config to prevent model_config.model_dump() errors - from lighteval.models.dummy.dummy_model import DummyModelConfig - - dummy_model_config = DummyModelConfig(model_name="test_model") - tracker.general_config_logger.log_model_info(model_config=dummy_model_config) - - yield tracker - - -@pytest.fixture -def mock_datetime(monkeypatch): - mock_date = datetime(2023, 1, 1, 12, 0, 0) - - class MockDatetime: - @classmethod - def now(cls): - return mock_date - - @classmethod - def fromisoformat(cls, date_string: str): - return mock_date - - monkeypatch.setattr("lighteval.logging.evaluation_tracker.datetime", MockDatetime) - return mock_date - - -class TestLogging: - def test_results_logging(self, mock_evaluation_tracker: EvaluationTracker): - task_metrics = { - "task1": {"accuracy": 0.8, "f1": 0.75}, - "task2": {"precision": 0.9, "recall": 0.85}, - } - mock_evaluation_tracker.metrics_logger.metric_aggregated = task_metrics - - mock_evaluation_tracker.save() - - results_dir = Path(mock_evaluation_tracker.output_dir) / "results" / "test_model" - assert results_dir.exists() - - result_files = list(results_dir.glob("results_*.json")) - assert len(result_files) == 1 - - with open(result_files[0], "r") as f: - saved_results = json.load(f) - - assert "results" in saved_results - assert saved_results["results"] == task_metrics - assert saved_results["config_general"]["model_name"] == "test_model" - - def test_results_logging_template(self, mock_evaluation_tracker: EvaluationTracker): - task_metrics = { - "task1": {"accuracy": 0.8, "f1": 0.75}, - "task2": {"precision": 0.9, "recall": 0.85}, - } - mock_evaluation_tracker.metrics_logger.metric_aggregated = task_metrics - mock_evaluation_tracker.results_path_template = "{output_dir}/{org}_{model}" - - mock_evaluation_tracker.save() - - results_dir = Path(mock_evaluation_tracker.output_dir) / "_test_model" - assert results_dir.exists() - - result_files = list(results_dir.glob("results_*.json")) - assert len(result_files) == 1 - - with open(result_files[0], "r") as f: - saved_results = json.load(f) - - assert "results" in saved_results - assert saved_results["results"] == task_metrics - assert saved_results["config_general"]["model_name"] == "test_model" - - @pytest.mark.evaluation_tracker(save_details=True) - def test_details_logging(self, mock_evaluation_tracker, mock_datetime): - task_details = { - "task1": [DetailsLogger.CompiledDetail(hashes=None, truncated=10, padded=5)], - "task2": [DetailsLogger.CompiledDetail(hashes=None, truncated=20, padded=10)], - } - mock_evaluation_tracker.details_logger.details = task_details - - mock_evaluation_tracker.save() - - date_id = mock_datetime.isoformat().replace(":", "-") - details_dir = Path(mock_evaluation_tracker.output_dir) / "details" / "test_model" / date_id - assert details_dir.exists() - - for task in ["task1", "task2"]: - file_path = details_dir / f"details_{task}_{date_id}.parquet" - dataset = Dataset.from_parquet(str(file_path)) - assert len(dataset) == 1 - assert int(dataset[0]["truncated"]) == task_details[task][0].truncated - assert int(dataset[0]["padded"]) == task_details[task][0].padded - - @pytest.mark.evaluation_tracker(save_details=False) - def test_no_details_output(self, mock_evaluation_tracker: EvaluationTracker): - mock_evaluation_tracker.save() - - details_dir = Path(mock_evaluation_tracker.output_dir) / "details" / "test_model" - assert not details_dir.exists() - - @pytest.mark.skip( # skipif - reason="Secrets are not available in this environment", - # condition=os.getenv("HF_TEST_TOKEN") is None, - ) - @pytest.mark.evaluation_tracker(push_to_hub=True, hub_results_org=TESTING_EMPTY_HF_ORG_ID) - def test_push_to_hub_works( - self, testing_empty_hf_org_id, mock_evaluation_tracker: EvaluationTracker, mock_datetime - ): - # Prepare the dummy data - task_metrics = { - "task1": {"accuracy": 0.8, "f1": 0.75}, - "task2": {"precision": 0.9, "recall": 0.85}, - } - mock_evaluation_tracker.metrics_logger.metric_aggregated = task_metrics - - task_details = { - "task1": [DetailsLogger.CompiledDetail(truncated=10, padded=5)], - "task2": [DetailsLogger.CompiledDetail(truncated=20, padded=10)], - } - mock_evaluation_tracker.details_logger.details = task_details - mock_evaluation_tracker.save() - - # Verify using HfApi - api = HfApi() - - # Check if repo exists and it's private - expected_repo_id = f"{testing_empty_hf_org_id}/details_test_model_private" - assert api.repo_exists(repo_id=expected_repo_id, repo_type="dataset") - assert api.repo_info(repo_id=expected_repo_id, repo_type="dataset").private - - repo_files = api.list_repo_files(repo_id=expected_repo_id, repo_type="dataset") - # Check if README.md exists - assert any(file == "README.md" for file in repo_files) - - # Check that both results files were uploaded - result_files = [file for file in repo_files if file.startswith("results_")] - assert len(result_files) == 2 - assert len([file for file in result_files if file.endswith(".json")]) == 1 - assert len([file for file in result_files if file.endswith(".parquet")]) == 1 - - # Check that the details dataset was uploaded - details_files = [file for file in repo_files if "details_" in file and file.endswith(".parquet")] - assert len(details_files) == 2 - - -class TestProperties(unittest.TestCase): - def setUp(self): - # In setup in case we need to reuse for future tests - from lighteval.models.dummy.dummy_model import DummyModelConfig - from lighteval.models.endpoints.endpoint_model import ( - ServerlessEndpointModelConfig, - InferenceEndpointModelConfig, - ) - from lighteval.models.endpoints.inference_providers_model import InferenceProvidersModelConfig - from lighteval.models.endpoints.litellm_model import LiteLLMModelConfig - from lighteval.models.endpoints.tgi_model import TGIModelConfig - from lighteval.models.sglang.sglang_model import SGLangModelConfig - from lighteval.models.transformers.transformers_model import TransformersModelConfig - from lighteval.models.transformers.vlm_transformers_model import VLMTransformersModelConfig - from lighteval.models.vllm.vllm_model import VLLMModelConfig - - # Tested model configurations - self.dummy_config = DummyModelConfig(model_name="test/case") - self.endpoint_serverless_config = ServerlessEndpointModelConfig(model_name="test/case") - self.endpoint_ie_config = InferenceEndpointModelConfig(model_name="test/case") - self.endpoint_ip_config = InferenceProvidersModelConfig(model_name="test/case", provider="no_provider") - self.endpoint_litellm_config = LiteLLMModelConfig(model_name="test/case") - self.tgi_config = TGIModelConfig(model_name="test/case") - self.sg_lang_config = SGLangModelConfig(model_name="test/case") - self.transformers_config = TransformersModelConfig(model_name="test/case") - self.vlm_transformers_config = VLMTransformersModelConfig(model_name="test/case") - self.vllm_config = VLLMModelConfig(model_name="test/case") - - # Reference configurations for expected results - ref_system_prompt = None - ref_generation_parameters = { - "num_blocks": None, - "block_size": None, - "early_stopping": None, - "repetition_penalty": None, - "frequency_penalty": None, - "length_penalty": None, - "presence_penalty": None, - "max_new_tokens": None, - "min_new_tokens": None, - "seed": None, - "stop_tokens": None, - "temperature": 0, - "top_k": None, - "min_p": None, - "top_p": None, - "truncate_prompt": None, - "cache_implementation": None, - "response_format": None, - } # ruff: noqa: E501 - self.dummy_ref_config = { - "model_name": "test/case", - "seed": 42, - "system_prompt": ref_system_prompt, - "generation_parameters": ref_generation_parameters, - } # ruff: noqa: E501 - self.endpoint_serverless_ref_config = { - "model_name": "test/case", - "add_special_tokens": True, - "batch_size": 1, - "system_prompt": ref_system_prompt, - "generation_parameters": ref_generation_parameters, - } # ruff: noqa: E501 - self.endpoint_ie_ref_config = { - "endpoint_name": None, - "model_name": "test/case", - "reuse_existing": False, - "accelerator": "gpu", - "dtype": None, - "vendor": "aws", - "region": "us-east-1", - "instance_size": None, - "instance_type": None, - "framework": "pytorch", - "endpoint_type": "protected", - "add_special_tokens": True, - "revision": "main", - "namespace": None, - "image_url": None, - "env_vars": None, - "batch_size": 1, - "system_prompt": ref_system_prompt, - "generation_parameters": ref_generation_parameters, - } # ruff: noqa: E501 - self.endpoint_ip_ref_config = { - "model_name": "test/case", - "provider": "no_provider", - "timeout": None, - "proxies": None, - "org_to_bill": None, - "parallel_calls_count": 10, - "system_prompt": ref_system_prompt, - "generation_parameters": ref_generation_parameters, - } # ruff: noqa: E501 - self.endpoint_litellm_ref_config = { - "model_name": "test/case", - "provider": None, - "base_url": None, - "api_key": None, - "system_prompt": ref_system_prompt, - "generation_parameters": ref_generation_parameters, - } # ruff: noqa: E501 - self.tgi_ref_config = { - "inference_server_address": None, - "inference_server_auth": None, - "model_name": "test/case", - "model_info": None, - "system_prompt": ref_system_prompt, - "generation_parameters": ref_generation_parameters, - } # ruff: noqa: E501 - self.sg_lang_ref_config = { - "model_name": "test/case", - "load_format": "auto", - "dtype": "auto", - "tp_size": 1, - "dp_size": 1, - "context_length": None, - "random_seed": 1234, - "trust_remote_code": False, - "device": "cuda", - "skip_tokenizer_init": False, - "kv_cache_dtype": "auto", - "add_special_tokens": True, - "pairwise_tokenization": False, - "sampling_backend": None, - "attention_backend": None, - "mem_fraction_static": 0.8, - "chunked_prefill_size": 4096, - "system_prompt": ref_system_prompt, - "generation_parameters": ref_generation_parameters, - } # ruff: noqa: E501 - self.transformers_ref_config = { - "model_name": "test/case", - "tokenizer": None, - "subfolder": None, - "revision": "main", - "batch_size": None, - "max_length": None, - "model_loading_kwargs": {}, - "add_special_tokens": True, - "skip_special_tokens": True, - "model_parallel": None, - "dtype": None, - "device": "cuda", - "trust_remote_code": False, - "compile": False, - "multichoice_continuations_start_space": None, - "pairwise_tokenization": False, - "continuous_batching": False, - "system_prompt": ref_system_prompt, - "generation_parameters": ref_generation_parameters, - } # ruff: noqa: E501 - self.vlm_transformers_ref_config = { - "model_name": "test/case", - "processor": None, - "use_fast_image_processor": None, - "subfolder": None, - "revision": "main", - "batch_size": 1, - "generation_size": None, - "max_length": None, - "add_special_tokens": True, - "model_parallel": None, - "dtype": None, - "device": "cuda", - "trust_remote_code": False, - "compile": False, - "device_map": None, - "system_prompt": ref_system_prompt, - "generation_parameters": ref_generation_parameters, - } # ruff: noqa: E501 - self.vllm_ref_config = { - "model_name": "test/case", - "revision": "main", - "dtype": "bfloat16", - "tensor_parallel_size": 1, - "data_parallel_size": 1, - "pipeline_parallel_size": 1, - "gpu_memory_utilization": 0.9, - "max_model_length": None, - "quantization": None, - "load_format": None, - "swap_space": 4, - "seed": 1234, - "trust_remote_code": False, - "add_special_tokens": True, - "multichoice_continuations_start_space": True, - "pairwise_tokenization": False, - "max_num_seqs": 128, - "max_num_batched_tokens": 2048, - "subfolder": None, - "is_async": False, - "system_prompt": ref_system_prompt, - "generation_parameters": ref_generation_parameters, - } # ruff: noqa: E501 - - def test_default_property_with_different_model_configs(self): - """Test that results property correctly handles different model configurations.""" - for model_config in [ - self.dummy_config, - self.endpoint_serverless_config, - self.endpoint_ie_config, - self.endpoint_ip_config, - self.endpoint_litellm_config, - self.tgi_config, - self.sg_lang_config, - self.transformers_config, - self.vlm_transformers_config, - self.vllm_config, - ]: - with self.subTest(model_config=model_config): - with tempfile.TemporaryDirectory() as tmp_dir: - evaluation_tracker = EvaluationTracker(output_dir=tmp_dir) - - evaluation_tracker.general_config_logger.log_model_info( - model_config=model_config, - ) - - results = evaluation_tracker.results - - # Verify structure - self.assertIsInstance(results, dict) - for key in [ - "config_general", - "results", - "versions", - "config_tasks", - "summary_tasks", - "summary_general", - ]: - self.assertIn(key, results.keys()) - - # Configs should all be empty since not initialized - self.assertEqual(results["versions"], {}) - self.assertEqual(results["config_tasks"], {}) - self.assertEqual(results["summary_tasks"], {}) - self.assertEqual( - results["summary_general"], - { - "hashes": {}, - "truncated": 0, - "non_truncated": 0, - "padded": 0, - "non_padded": 0, - }, - ) - - # Except config_general, which should contain the model config among other things - general_config = results["config_general"] - # We skip testing lighteval_sha, start_time - self.assertIsNone(general_config["num_fewshot_seeds"]) - self.assertIsNone(general_config["max_samples"]) - self.assertIsNone(general_config["job_id"]) - self.assertIsNone(general_config["end_time"]) - self.assertIsNone(general_config["total_evaluation_time_secondes"]) - self.assertEqual(general_config["model_name"], "test/case") - - def test_model_config_property_with_different_model_configs(self): - """Test that the model configs are properly saved.""" - for model_config, ref_config in [ - (self.dummy_config, self.dummy_ref_config), - (self.endpoint_serverless_config, self.endpoint_serverless_ref_config), - (self.endpoint_ie_config, self.endpoint_ie_ref_config), - (self.endpoint_ip_config, self.endpoint_ip_ref_config), - (self.endpoint_litellm_config, self.endpoint_litellm_ref_config), - (self.tgi_config, self.tgi_ref_config), - (self.sg_lang_config, self.sg_lang_ref_config), - (self.transformers_config, self.transformers_ref_config), - (self.vlm_transformers_config, self.vlm_transformers_ref_config), - (self.vllm_config, self.vllm_ref_config), - ]: - with tempfile.TemporaryDirectory() as tmp_dir: - evaluation_tracker = EvaluationTracker(output_dir=tmp_dir) - - evaluation_tracker.general_config_logger.log_model_info( - model_config=model_config, - ) - - results = evaluation_tracker.results - - # Now to the core test, the model_config - for k, v in ref_config.items(): - with self.subTest(model_config=model_config, model_property=k): - self.assertEqual(results["config_general"]["model_config"][k], v) diff --git a/tests/unit/metrics/pytest.ini b/tests/unit/metrics/pytest.ini deleted file mode 100644 index f5198f45c..000000000 --- a/tests/unit/metrics/pytest.ini +++ /dev/null @@ -1,18 +0,0 @@ -[tool:pytest] -testpaths = . -python_files = test_*.py -python_classes = Test* -python_functions = test_* -addopts = - -v - --tb=short - --strict-markers - --disable-warnings -markers = - slow: marks tests as slow (deselect with '-m "not slow"') - unit: marks tests as unit tests - integration: marks tests as integration tests - automated: marks tests as automated metric tests -filterwarnings = - ignore::DeprecationWarning - ignore::PendingDeprecationWarning diff --git a/tests/unit/metrics/test_automated_metrics_pytest.py b/tests/unit/metrics/test_automated_metrics_pytest.py deleted file mode 100644 index eb441e3bc..000000000 --- a/tests/unit/metrics/test_automated_metrics_pytest.py +++ /dev/null @@ -1,104 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -""" -Pytest integration for the automated metric testing framework. - -This module provides pytest fixtures and test functions that can load and run -test cases from JSON files. -""" - -import json -from pathlib import Path -from typing import List - -import pytest -from test_metrics_automated import AutomatedMetricTester, MetricTestSuite - - -@pytest.fixture -def metric_tester(): - """Fixture providing an AutomatedMetricTester instance.""" - return AutomatedMetricTester() - - -def load_test_suite_from_file(file_path: str) -> MetricTestSuite: - """Load a test suite from a JSON file.""" - with open(file_path, "r") as f: - data = json.load(f) - return MetricTestSuite(**data) - - -def get_test_suite_files() -> List[str]: - """Get all test suite JSON files from the test_cases directory.""" - test_cases_dir = Path(__file__).parent / "test_cases" - if not test_cases_dir.exists(): - return [] - - json_files = list(test_cases_dir.glob("*.json")) - return [str(f) for f in json_files] - - -def parametrize_test_suites(): - """Create parametrized test cases for all test suite files.""" - test_files = get_test_suite_files() - if not test_files: - pytest.skip("No test suite files found") - - return test_files - - -class TestAutomatedMetrics: - """Test class for automated metric testing with pytest.""" - - @pytest.mark.parametrize("test_file", parametrize_test_suites()) - def test_metric_suite(self, metric_tester, test_file): - """Test a complete metric test suite from a JSON file.""" - test_suite = load_test_suite_from_file(test_file) - - # Run all test cases in the suite - results = metric_tester.run_test_suite(test_suite) - - # Separate failed tests from skipped tests - failed_tests = [r for r in results if not r["success"] and not r.get("skipped", False)] - skipped_tests = [r for r in results if r.get("skipped", False)] - - if failed_tests: - # Create detailed error message - error_msg = f"Test suite '{test_suite.name}' failed with {len(failed_tests)} failed tests:\n" - for result in failed_tests: - error_msg += f"\n - {result['test_case']}: " - if result["error"]: - error_msg += f"Error: {result['error']}" - else: - error_msg += f"Expected {result['expected']}, got {result['actual']}" - - pytest.fail(error_msg) - - # Log skipped tests - if skipped_tests: - print(f"\nSkipped {len(skipped_tests)} tests in '{test_suite.name}':") - for result in skipped_tests: - print(f" - {result['test_case']}: {result.get('skip_reason', 'Unknown reason')}") - - # All non-skipped tests passed - assert len(failed_tests) == 0, f"Expected all non-skipped tests to pass, but {len(failed_tests)} failed" diff --git a/tests/unit/metrics/test_cases/acc_golds_likelihood.json b/tests/unit/metrics/test_cases/acc_golds_likelihood.json deleted file mode 100644 index 90a37d8cf..000000000 --- a/tests/unit/metrics/test_cases/acc_golds_likelihood.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "name": "Acc Golds Likelihood Test Suite", - "description": "Test cases for acc_golds_likelihood metric", - "test_cases": [ - { - "name": "Acc Golds Likelihood - Correct Likelihood", - "metric_class": "acc_golds_likelihood", - "metric_params": {}, - "doc": { - "query": "What is the capital of France?", - "choices": ["Paris", "London", "Berlin"], - "gold_index": 0, - "task_name": "geography" - }, - "model_response": { - "argmax_logits_eq_gold": [1, 0, 0] - }, - "expected_output": { - "acc": 1 - }, - "tolerance": 0.01, - "description": "Test acc golds likelihood with correct likelihood" - }, - { - "name": "Acc Golds Likelihood - Incorrect Likelihood", - "metric_class": "acc_golds_likelihood", - "metric_params": {}, - "doc": { - "query": "What is the capital of France?", - "choices": ["Paris", "London", "Berlin"], - "gold_index": 0, - "task_name": "geography" - }, - "model_response": { - "argmax_logits_eq_gold": [0, 0, 0] - }, - "expected_output": { - "acc": 0 - }, - "tolerance": 0.01, - "description": "Test acc golds likelihood with incorrect likelihood" - } - ] -} diff --git a/tests/unit/metrics/test_cases/avg_at_k.json b/tests/unit/metrics/test_cases/avg_at_k.json deleted file mode 100644 index 907345b56..000000000 --- a/tests/unit/metrics/test_cases/avg_at_k.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "name": "Avg At K Test Suite", - "description": "Test cases for avg_at_k metric", - "test_cases": [ - { - "name": "Avg at K - Correct in Top K", - "metric_class": "avg_at_k", - "metric_params": {"k": 2}, - "doc": { - "query": "What is the capital of France?", - "choices": ["London", "Paris", "Berlin"], - "gold_index": 1, - "task_name": "geography" - }, - "model_response": { - "text": ["Paris", "London", "Berlin"] - }, - "expected_output": { - "avg@k:k=2": 0.5 - }, - "tolerance": 0.01, - "description": "Test avg at k with correct answer in top k" - }, - { - "name": "Avg at K - Not in Top K", - "metric_class": "avg_at_k", - "metric_params": {"k": 1}, - "doc": { - "query": "What is the capital of France?", - "choices": ["London", "Paris", "Berlin"], - "gold_index": 1, - "task_name": "geography" - }, - "model_response": { - "text": ["London", "Berlin", "Paris"] - }, - "expected_output": { - "avg@k:k=1": 0.0 - }, - "tolerance": 0.01, - "description": "Test avg at k with correct answer not in top k" - }, - { - "name": "Avg at K - Multiple Correct", - "metric_class": "avg_at_k", - "metric_params": {"k": 3}, - "doc": { - "query": "Which are European capitals?", - "choices": ["London", "Paris", "Tokyo", "Berlin"], - "gold_index": [0, 1, 3], - "task_name": "geography" - }, - "model_response": { - "text": ["Paris", "London", "Berlin", "Tokyo"] - }, - "expected_output": { - "avg@k:k=3": 0.33 - }, - "tolerance": 0.01, - "description": "Test avg at k with multiple correct answers" - } - ] -} diff --git a/tests/unit/metrics/test_cases/avg_at_k_math.json b/tests/unit/metrics/test_cases/avg_at_k_math.json deleted file mode 100644 index 60a712ea0..000000000 --- a/tests/unit/metrics/test_cases/avg_at_k_math.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "name": "Avg At K Math Test Suite", - "description": "Test cases for avg_at_k_math metric", - "test_cases": [ - { - "name": "Avg at K Math - Correct Math", - "metric_class": "avg_at_k_math", - "metric_params": {"k": 1}, - "doc": { - "query": "What is 2 + 2?", - "choices": ["4"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["4"] - }, - "expected_output": { - "avg@k:k=1": 1.0 - }, - "tolerance": 0.01, - "description": "Test avg at k math with correct math answer" - }, - { - "name": "Avg at K Math - Wrong Math", - "metric_class": "avg_at_k_math", - "metric_params": {"k": 1}, - "doc": { - "query": "What is 2 + 2?", - "choices": ["4"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["5"] - }, - "expected_output": { - "avg@k:k=1": 0.0 - }, - "tolerance": 0.01, - "description": "Test avg at k math with wrong math answer" - }, - { - "name": "Avg at K Math - Multiple Attempts", - "metric_class": "avg_at_k_math", - "metric_params": {"k": 2}, - "doc": { - "query": "What is 3 * 4?", - "choices": ["12"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["12", "15"] - }, - "expected_output": { - "avg@k:k=2": 0.5 - }, - "tolerance": 0.01, - "description": "Test avg at k math with multiple attempts" - } - ] -} diff --git a/tests/unit/metrics/test_cases/bert_score.json b/tests/unit/metrics/test_cases/bert_score.json deleted file mode 100644 index 13cda7625..000000000 --- a/tests/unit/metrics/test_cases/bert_score.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "name": "Bert Score Test Suite", - "description": "Test cases for bert_score metric", - "test_cases": [ - { - "name": "Bert Score - Basic Test", - "metric_class": "bert_score", - "metric_params": {}, - "doc": { - "query": "Test query for bert_score", - "choices": [ - "Test choice 1", - "Test choice 2", - "Test choice 3" - ], - "gold_index": 0, - "task_name": "test" - }, - "model_response": { - "text": [ - "Test choice 1" - ], - "logprobs": [ - 0.5, - 0.3, - 0.2 - ], - "output_tokens": [ - [ - 1 - ], - [ - 2 - ], - [ - 3 - ] - ] - }, - "expected_output": { - "result": 1.0 - }, - "tolerance": 0.01, - "description": "Basic test case for bert_score metric" - } - ] -} diff --git a/tests/unit/metrics/test_cases/bits_per_byte.json b/tests/unit/metrics/test_cases/bits_per_byte.json deleted file mode 100644 index 8470678fa..000000000 --- a/tests/unit/metrics/test_cases/bits_per_byte.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "name": "Bits Per Byte Test Suite", - "description": "Test cases for bits_per_byte metric", - "test_cases": [ - { - "name": "Bits Per Byte - Basic Test", - "metric_class": "bits_per_byte", - "metric_params": {}, - "doc": { - "query": "Test query for bits_per_byte", - "choices": [ - "Test choice 1", - "Test choice 2", - "Test choice 3" - ], - "gold_index": 0, - "task_name": "test" - }, - "model_response": { - "text": [ - "Test choice 1" - ], - "logprobs": [ - 0.5, - 0.3, - 0.2 - ], - "output_tokens": [ - [ - 1 - ], - [ - 2 - ], - [ - 3 - ] - ] - }, - "expected_output": { - "bits_per_byte": 1.0 - }, - "tolerance": 0.01, - "description": "Basic test case for bits_per_byte metric" - } - ] -} diff --git a/tests/unit/metrics/test_cases/bleu.json b/tests/unit/metrics/test_cases/bleu.json deleted file mode 100644 index fb8ebbfc4..000000000 --- a/tests/unit/metrics/test_cases/bleu.json +++ /dev/null @@ -1,167 +0,0 @@ -{ - "name": "BLEU Test Suite", - "description": "Test cases for bleu metric (corpus-level BLEU)", - "corpus_level": true, - "test_cases": [ - { - "name": "BLEU - Perfect Translations", - "metric_class": "bleu", - "metric_params": {}, - "metric_name": "bleu", - "docs": [ - { - "query": "Translate to French: The beautiful flowers are blooming in the garden today", - "choices": ["Les belles fleurs fleurissent dans le jardin aujourd'hui"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to Spanish: My family and I went to the beach last weekend", - "choices": ["Mi familia y yo fuimos a la playa el fin de semana pasado"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to German: The children are playing with their new toys in the park", - "choices": ["Die Kinder spielen mit ihren neuen Spielzeugen im Park"], - "gold_index": 0, - "task_name": "translation" - } - ], - "model_responses": [ - { - "text": ["Les belles fleurs fleurissent dans le jardin aujourd'hui"] - }, - { - "text": ["Mi familia y yo fuimos a la playa el fin de semana pasado"] - }, - { - "text": ["Die Kinder spielen mit ihren neuen Spielzeugen im Park"] - } - ], - "expected_output": 100.0, - "tolerance": 0.01, - "description": "Perfect translations - exact word overlap (BLEU = 100.0)" - }, - { - "name": "BLEU - High Similarity", - "metric_class": "bleu", - "metric_params": {}, - "metric_name": "bleu", - "docs": [ - { - "query": "Translate to French: The cat is sleeping", - "choices": ["Le chat dort"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to Spanish: I like pizza", - "choices": ["Me gusta la pizza"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to German: The weather is nice", - "choices": ["Das Wetter ist schön"], - "gold_index": 0, - "task_name": "translation" - } - ], - "model_responses": [ - { - "text": ["Le chat dort"] - }, - { - "text": ["Me gusta pizza"] - }, - { - "text": ["Das Wetter ist schön"] - } - ], - "expected_output": 81.02, - "tolerance": 0.01, - "description": "High similarity - minor word differences (BLEU ≈ 85.0)" - }, - { - "name": "BLEU - Moderate Similarity", - "metric_class": "bleu", - "metric_params": {}, - "metric_name": "bleu", - "docs": [ - { - "query": "Translate to French: The quick brown fox jumped gracefully over the lazy sleeping dog", - "choices": ["Le renard brun rapide a sauté gracieusement par-dessus le chien paresseux endormi"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to Spanish: Artificial intelligence is revolutionizing the way we interact with technology", - "choices": ["La inteligencia artificial está revolucionando la forma en que interactuamos con la tecnología"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to German: Machine learning algorithms can analyze complex patterns in large datasets", - "choices": ["Maschinelle Lernalgorithmen können komplexe Muster in großen Datensätzen analysieren"], - "gold_index": 0, - "task_name": "translation" - } - ], - "model_responses": [ - { - "text": ["Le renard rapide a sauté par-dessus le chien"] - }, - { - "text": ["La IA revoluciona la tecnología"] - }, - { - "text": ["ML analysiert Daten"] - } - ], - "expected_output": 0.0, - "tolerance": 10.0, - "description": "Moderate similarity - significant word omissions (BLEU ≈ 45.0)" - }, - { - "name": "BLEU - Low Similarity", - "metric_class": "bleu", - "metric_params": {}, - "metric_name": "bleu", - "docs": [ - { - "query": "Translate to French: The bright sun shines warmly through the scattered clouds in the azure summer sky", - "choices": ["Le soleil brillant brille chaudement à travers les nuages épars dans le ciel bleu azur d'été"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to Spanish: The full moon casts mysterious shadows across the tranquil lake at midnight", - "choices": ["La luna llena proyecta sombras misteriosas sobre el lago tranquilo a medianoche"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to German: The twinkling stars illuminate the dark velvet sky like scattered diamonds", - "choices": ["Die funkelnden Sterne erleuchten den dunklen Samthimmel wie verstreute Diamanten"], - "gold_index": 0, - "task_name": "translation" - } - ], - "model_responses": [ - { - "text": ["Il fait mauvais temps aujourd'hui et le ciel est couvert"] - }, - { - "text": ["Las montañas son muy altas y majestuosas"] - }, - { - "text": ["Der Wind weht stark durch die Bäume"] - } - ], - "expected_output": 0.0, - "tolerance": 10.0, - "description": "Low similarity - minimal word overlap (BLEU ≈ 15.0)" - } - ] -} diff --git a/tests/unit/metrics/test_cases/bleu_1.json b/tests/unit/metrics/test_cases/bleu_1.json deleted file mode 100644 index 05dd676af..000000000 --- a/tests/unit/metrics/test_cases/bleu_1.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "name": "BLEU-1 Test Suite", - "description": "Test cases for bleu_1 metric (sample-level BLEU-1 with 1-gram overlap)", - "test_cases": [ - { - "name": "BLEU-1 - Perfect Match", - "metric_class": "bleu_1", - "metric_params": {}, - "doc": { - "query": "Translate to French: Hello world", - "choices": ["Bonjour le monde"], - "gold_index": 0, - "task_name": "translation" - }, - "model_response": { - "text": ["Bonjour le monde"] - }, - "expected_output": { - "bleu_1": 1.0 - }, - "tolerance": 0.01, - "description": "Perfect match - exact 1-gram overlap (BLEU-1 = 1.0)" - }, - { - "name": "BLEU-1 - High Similarity", - "metric_class": "bleu_1", - "metric_params": {}, - "doc": { - "query": "Translate to French: The cat is sleeping", - "choices": ["Le chat dort"], - "gold_index": 0, - "task_name": "translation" - }, - "model_response": { - "text": ["Le chat dort"] - }, - "expected_output": { - "bleu_1": 1.0 - }, - "tolerance": 0.01, - "description": "High similarity - exact 1-gram match (BLEU-1 = 1.0)" - }, - { - "name": "BLEU-1 - Partial Match", - "metric_class": "bleu_1", - "metric_params": {}, - "doc": { - "query": "Translate to French: The quick brown fox", - "choices": ["Le renard brun rapide"], - "gold_index": 0, - "task_name": "translation" - }, - "model_response": { - "text": ["Le renard rapide"] - }, - "expected_output": { - "bleu_1": 0.75 - }, - "tolerance": 0.1, - "description": "Partial match - 3 out of 4 words match (BLEU-1 = 0.75)" - }, - { - "name": "BLEU-1 - Low Similarity", - "metric_class": "bleu_1", - "metric_params": {}, - "doc": { - "query": "Translate to French: The sun is bright", - "choices": ["Le soleil est brillant"], - "gold_index": 0, - "task_name": "translation" - }, - "model_response": { - "text": ["Il pleut"] - }, - "expected_output": { - "bleu_1": 0.0 - }, - "tolerance": 0.01, - "description": "Low similarity - no 1-gram overlap (BLEU-1 = 0.0)" - }, - { - "name": "BLEU-1 - Word Order Change", - "metric_class": "bleu_1", - "metric_params": {}, - "doc": { - "query": "Translate to French: The weather is nice", - "choices": ["Le temps est agréable"], - "gold_index": 0, - "task_name": "translation" - }, - "model_response": { - "text": ["Le agréable temps est"] - }, - "expected_output": { - "bleu_1": 1.0 - }, - "tolerance": 0.01, - "description": "Word order change - same 1-grams, different order (BLEU-1 = 1.0)" - } - ] -} diff --git a/tests/unit/metrics/test_cases/bleu_4.json b/tests/unit/metrics/test_cases/bleu_4.json deleted file mode 100644 index e6e8d2814..000000000 --- a/tests/unit/metrics/test_cases/bleu_4.json +++ /dev/null @@ -1,120 +0,0 @@ -{ - "name": "BLEU-4 Test Suite", - "description": "Test cases for bleu_4 metric (sample-level BLEU-4 with 4-gram overlap)", - "test_cases": [ - { - "name": "BLEU-4 - Perfect Match", - "metric_class": "bleu_4", - "metric_params": {}, - "doc": { - "query": "Translate to French: The quick brown fox jumps", - "choices": ["Le renard brun rapide saute"], - "gold_index": 0, - "task_name": "translation" - }, - "model_response": { - "text": ["Le renard brun rapide saute"] - }, - "expected_output": { - "bleu_4": 1.0 - }, - "tolerance": 0.01, - "description": "Perfect match - exact 4-gram overlap (BLEU-4 = 1.0)" - }, - { - "name": "BLEU-4 - High Similarity", - "metric_class": "bleu_4", - "metric_params": {}, - "doc": { - "query": "Translate to French: The cat is sleeping now", - "choices": ["Le chat dort maintenant"], - "gold_index": 0, - "task_name": "translation" - }, - "model_response": { - "text": ["Le chat dort maintenant"] - }, - "expected_output": { - "bleu_4": 1.0 - }, - "tolerance": 0.01, - "description": "High similarity - exact 4-gram match (BLEU-4 = 1.0)" - }, - { - "name": "BLEU-4 - Partial Match", - "metric_class": "bleu_4", - "metric_params": {}, - "doc": { - "query": "Translate to French: The weather is very nice", - "choices": ["Le temps est très agréable"], - "gold_index": 0, - "task_name": "translation" - }, - "model_response": { - "text": ["Le temps est agréable"] - }, - "expected_output": { - "bleu_4": 0.0 - }, - "tolerance": 0.1, - "description": "Partial match - some 4-grams match (BLEU-4 = 0.6)" - }, - { - "name": "BLEU-4 - Low Similarity", - "metric_class": "bleu_4", - "metric_params": {}, - "doc": { - "query": "Translate to French: The sun is bright today", - "choices": ["Le soleil est brillant aujourd'hui"], - "gold_index": 0, - "task_name": "translation" - }, - "model_response": { - "text": ["Il pleut beaucoup"] - }, - "expected_output": { - "bleu_4": 0.0 - }, - "tolerance": 0.01, - "description": "Low similarity - no 4-gram overlap (BLEU-4 = 0.0)" - }, - { - "name": "BLEU-4 - Word Order Change", - "metric_class": "bleu_4", - "metric_params": {}, - "doc": { - "query": "Translate to French: The weather is nice today", - "choices": ["Le temps est agréable aujourd'hui"], - "gold_index": 0, - "task_name": "translation" - }, - "model_response": { - "text": ["Le agréable temps est aujourd'hui"] - }, - "expected_output": { - "bleu_4": 0.0 - }, - "tolerance": 0.01, - "description": "Word order change - no 4-gram matches (BLEU-4 = 0.0)" - }, - { - "name": "BLEU-4 - Short Text", - "metric_class": "bleu_4", - "metric_params": {}, - "doc": { - "query": "Translate to French: Hello", - "choices": ["Bonjour"], - "gold_index": 0, - "task_name": "translation" - }, - "model_response": { - "text": ["Bonjour"] - }, - "expected_output": { - "bleu_4": 0.0 - }, - "tolerance": 0.01, - "description": "Short text - single word, BLEU-4 defaults to BLEU-1 (BLEU-4 = 1.0)" - } - ] -} diff --git a/tests/unit/metrics/test_cases/bleurt.json b/tests/unit/metrics/test_cases/bleurt.json deleted file mode 100644 index 7891b2aec..000000000 --- a/tests/unit/metrics/test_cases/bleurt.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "name": "Bleurt Test Suite", - "description": "Test cases for bleurt metric", - "test_cases": [ - { - "name": "BLEURT - Perfect Match", - "metric_class": "bleurt", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["The quick brown fox jumps over the lazy dog"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "bleurt": 0.82 - }, - "tolerance": 0.1, - "description": "Test BLEURT with perfect match" - }, - { - "name": "BLEURT - Partial Match", - "metric_class": "bleurt", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["The quick brown fox"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "bleurt": -0.14 - }, - "tolerance": 0.2, - "description": "Test BLEURT with partial match" - }, - { - "name": "BLEURT - Different Content", - "metric_class": "bleurt", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["A cat sleeps on the mat"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "bleurt": -1.11 - }, - "tolerance": 0.2, - "description": "Test BLEURT with completely different content" - } - ] -} diff --git a/tests/unit/metrics/test_cases/byte_perplexity.json b/tests/unit/metrics/test_cases/byte_perplexity.json deleted file mode 100644 index ef76f6bb7..000000000 --- a/tests/unit/metrics/test_cases/byte_perplexity.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "name": "Byte Perplexity Test Suite", - "description": "Test cases for byte_perplexity metric", - "test_cases": [ - { - "name": "Byte Perplexity - Basic Test", - "metric_class": "byte_perplexity", - "metric_params": {}, - "doc": { - "query": "Test query for byte_perplexity", - "choices": [ - "Test choice 1", - "Test choice 2", - "Test choice 3" - ], - "gold_index": 0, - "task_name": "test" - }, - "model_response": { - "text": [ - "Test choice 1" - ], - "logprobs": [ - 0.5, - 0.3, - 0.2 - ], - "output_tokens": [ - [ - 1 - ], - [ - 2 - ], - [ - 3 - ] - ] - }, - "expected_output": { - "byte_perplexity": 1.0 - }, - "tolerance": 0.01, - "description": "Basic test case for byte_perplexity metric" - } - ] -} diff --git a/tests/unit/metrics/test_cases/chrf.json b/tests/unit/metrics/test_cases/chrf.json deleted file mode 100644 index f55028674..000000000 --- a/tests/unit/metrics/test_cases/chrf.json +++ /dev/null @@ -1,207 +0,0 @@ -{ - "name": "CHRF Test Suite", - "description": "Test cases for chrf metric (corpus-level CHRF without word order)", - "corpus_level": true, - "test_cases": [ - { - "name": "CHRF - Perfect Matches", - "metric_class": "chrf", - "metric_params": {}, - "metric_name": "chrf", - "docs": [ - { - "query": "Translate to French: Hello world", - "choices": ["Bonjour le monde"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to Spanish: Good morning", - "choices": ["Buenos días"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to German: Thank you", - "choices": ["Danke schön"], - "gold_index": 0, - "task_name": "translation" - } - ], - "model_responses": [ - { - "text": ["Bonjour le monde"] - }, - { - "text": ["Buenos días"] - }, - { - "text": ["Danke schön"] - } - ], - "expected_output": 100.0, - "tolerance": 0.1, - "description": "Perfect matches - exact character overlap (CHRF = 100.0)" - }, - { - "name": "CHRF - High Similarity", - "metric_class": "chrf", - "metric_params": {}, - "metric_name": "chrf", - "docs": [ - { - "query": "Translate to French: The cat is sleeping", - "choices": ["Le chat dort"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to Spanish: I like pizza", - "choices": ["Me gusta la pizza"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to German: The weather is nice", - "choices": ["Das Wetter ist schön"], - "gold_index": 0, - "task_name": "translation" - } - ], - "model_responses": [ - { - "text": ["Le chat dort"] - }, - { - "text": ["Me gusta pizza"] - }, - { - "text": ["Das Wetter ist schön"] - } - ], - "expected_output": 100.0, - "tolerance": 0.1, - "description": "High similarity - minor character differences (CHRF ≈ 88.0)" - }, - { - "name": "CHRF - Word Order Changes", - "metric_class": "chrf", - "metric_params": {}, - "metric_name": "chrf", - "docs": [ - { - "query": "Translate to French: The quick brown fox", - "choices": ["Le renard brun rapide"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to Spanish: Artificial intelligence", - "choices": ["La inteligencia artificial"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to German: Machine learning", - "choices": ["Maschinelles Lernen"], - "gold_index": 0, - "task_name": "translation" - } - ], - "model_responses": [ - { - "text": ["Le rapide renard brun"] - }, - { - "text": ["La artificial inteligencia"] - }, - { - "text": ["Lernen Maschinelles"] - } - ], - "expected_output": 78.84, - "tolerance": 0.1, - "description": "Word order changes - same characters, different order (CHRF ≈ 75.0)" - }, - { - "name": "CHRF - Moderate Similarity", - "metric_class": "chrf", - "metric_params": {}, - "metric_name": "chrf", - "docs": [ - { - "query": "Translate to French: The sun is bright", - "choices": ["Le soleil est brillant"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to Spanish: The moon is full", - "choices": ["La luna está llena"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to German: The stars are beautiful", - "choices": ["Die Sterne sind wunderschön"], - "gold_index": 0, - "task_name": "translation" - } - ], - "model_responses": [ - { - "text": ["Le soleil"] - }, - { - "text": ["La luna"] - }, - { - "text": ["Die Sterne"] - } - ], - "expected_output": 37.68, - "tolerance": 0.1, - "description": "Moderate similarity - partial character overlap (CHRF ≈ 50.0)" - }, - { - "name": "CHRF - Low Similarity", - "metric_class": "chrf", - "metric_params": {}, - "metric_name": "chrf", - "docs": [ - { - "query": "Translate to French: The weather is nice", - "choices": ["Le temps est agréable"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to Spanish: The food is delicious", - "choices": ["La comida está deliciosa"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to German: The music is beautiful", - "choices": ["Die Musik ist wunderschön"], - "gold_index": 0, - "task_name": "translation" - } - ], - "model_responses": [ - { - "text": ["Il pleut beaucoup"] - }, - { - "text": ["Hace mucho frío"] - }, - { - "text": ["Es sehr heiß"] - } - ], - "expected_output": 7.7, - "tolerance": 0.1, - "description": "Low similarity - minimal character overlap (CHRF ≈ 20.0)" - } - ] -} diff --git a/tests/unit/metrics/test_cases/chrf_plus.json b/tests/unit/metrics/test_cases/chrf_plus.json deleted file mode 100644 index 29c45720d..000000000 --- a/tests/unit/metrics/test_cases/chrf_plus.json +++ /dev/null @@ -1,167 +0,0 @@ -{ - "name": "CHRF Plus Test Suite", - "description": "Test cases for chrf_plus metric (corpus-level CHRF++ with word order)", - "corpus_level": true, - "test_cases": [ - { - "name": "CHRF Plus - Perfect Matches", - "metric_class": "chrf_plus", - "metric_params": {}, - "metric_name": "chrf++", - "docs": [ - { - "query": "Translate to French: Hello world", - "choices": ["Bonjour le monde"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to Spanish: Good morning", - "choices": ["Buenos días"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to German: Thank you", - "choices": ["Danke schön"], - "gold_index": 0, - "task_name": "translation" - } - ], - "model_responses": [ - { - "text": ["Bonjour le monde"] - }, - { - "text": ["Buenos días"] - }, - { - "text": ["Danke schön"] - } - ], - "expected_output": 100.0, - "tolerance": 0.01, - "description": "Perfect matches - exact character and word order overlap (CHRF++ = 100.0)" - }, - { - "name": "CHRF Plus - High Similarity", - "metric_class": "chrf_plus", - "metric_params": {}, - "metric_name": "chrf++", - "docs": [ - { - "query": "Translate to French: The cat is sleeping", - "choices": ["Le chat dort"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to Spanish: I like pizza", - "choices": ["Me gusta la pizza"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to German: The weather is nice", - "choices": ["Das Wetter ist schön"], - "gold_index": 0, - "task_name": "translation" - } - ], - "model_responses": [ - { - "text": ["Le chat dort"] - }, - { - "text": ["Me gusta pizza"] - }, - { - "text": ["Das Wetter ist schön"] - } - ], - "expected_output": 100.0, - "tolerance": 0.1, - "description": "High similarity - minor character differences (CHRF++ ≈ 85.0)" - }, - { - "name": "CHRF Plus - Moderate Similarity", - "metric_class": "chrf_plus", - "metric_params": {}, - "metric_name": "chrf++", - "docs": [ - { - "query": "Translate to French: The quick brown fox", - "choices": ["Le renard brun rapide"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to Spanish: Artificial intelligence", - "choices": ["La inteligencia artificial"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to German: Machine learning", - "choices": ["Maschinelles Lernen"], - "gold_index": 0, - "task_name": "translation" - } - ], - "model_responses": [ - { - "text": ["Le renard rapide"] - }, - { - "text": ["La IA"] - }, - { - "text": ["ML"] - } - ], - "expected_output": 58.82, - "tolerance": 0.1, - "description": "Moderate similarity - significant character omissions (CHRF++ ≈ 45.0)" - }, - { - "name": "CHRF Plus - Low Similarity", - "metric_class": "chrf_plus", - "metric_params": {}, - "metric_name": "chrf++", - "docs": [ - { - "query": "Translate to French: The sun is bright", - "choices": ["Le soleil est brillant"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to Spanish: The moon is full", - "choices": ["La luna está llena"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to German: The stars are beautiful", - "choices": ["Die Sterne sind wunderschön"], - "gold_index": 0, - "task_name": "translation" - } - ], - "model_responses": [ - { - "text": ["Il pleut"] - }, - { - "text": ["Hace frío"] - }, - { - "text": ["Es heiß"] - } - ], - "expected_output": 15.0, - "tolerance": 10.0, - "description": "Low similarity - minimal character overlap (CHRF++ ≈ 15.0)" - } - ] -} diff --git a/tests/unit/metrics/test_cases/copyright.json b/tests/unit/metrics/test_cases/copyright.json deleted file mode 100644 index fb5b434f4..000000000 --- a/tests/unit/metrics/test_cases/copyright.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "name": "Copyright Test Suite", - "description": "Test cases for copyright metric", - "test_cases": [ - { - "name": "Copyright - No Copyright", - "metric_class": "copyright", - "metric_params": {}, - "doc": { - "query": "Write a story", - "choices": ["Once upon a time"], - "gold_index": 0, - "task_name": "storytelling" - }, - "model_response": { - "text": ["Once upon a time"] - }, - "expected_output": { - "longest_common_prefix_length": 4.0, - "edit_distance": 0.0, - "edit_similarity": 1.0 - }, - "tolerance": 0.01, - "description": "Test copyright with no copyright violation" - }, - { - "name": "Copyright - Partial Match", - "metric_class": "copyright", - "metric_params": {}, - "doc": { - "query": "Write a story", - "choices": ["Once upon a time there was a princess"], - "gold_index": 0, - "task_name": "storytelling" - }, - "model_response": { - "text": ["Once upon a time there was a dragon"] - }, - "expected_output": { - "longest_common_prefix_length": 7.0, - "edit_distance": 1.0, - "edit_similarity": 0.875 - }, - "tolerance": 0.1, - "description": "Test copyright with partial match" - }, - { - "name": "Copyright - High Similarity", - "metric_class": "copyright", - "metric_params": {}, - "doc": { - "query": "Write a story", - "choices": ["Once upon a time there was a beautiful princess who lived in a castle"], - "gold_index": 0, - "task_name": "storytelling" - }, - "model_response": { - "text": ["Once upon a time there was a beautiful princess who lived in a palace"] - }, - "expected_output": { - "longest_common_prefix_length": 13.0, - "edit_distance": 1.0, - "edit_similarity": 0.923 - }, - "tolerance": 0.1, - "description": "Test copyright with high similarity" - } - ] -} diff --git a/tests/unit/metrics/test_cases/drop.json b/tests/unit/metrics/test_cases/drop.json deleted file mode 100644 index 49984c291..000000000 --- a/tests/unit/metrics/test_cases/drop.json +++ /dev/null @@ -1,75 +0,0 @@ -{ - "name": "Drop Test Suite", - "description": "Test cases for drop metric", - "test_cases": [ - { - "name": "DROP - Correct Answer", - "metric_class": "drop", - "metric_params": {}, - "doc": { - "query": "What is 2 + 2?", - "specific": { - "golds_no_preprocessing": ["4"] - }, - "choices": ["4"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["4"] - }, - "expected_output": { - "em": 1.0, - "f1": 1.0 - }, - "tolerance": 0.01, - "description": "Test DROP with correct answer" - }, - { - "name": "DROP - Wrong Answer", - "metric_class": "drop", - "metric_params": {}, - "doc": { - "query": "What is 2 + 2?", - "specific": { - "golds_no_preprocessing": ["4"] - }, - "choices": ["4"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["5"] - }, - "expected_output": { - "em": 0.0, - "f1": 0.0 - }, - "tolerance": 0.01, - "description": "Test DROP with wrong answer" - }, - { - "name": "DROP - Partial Match", - "metric_class": "drop", - "metric_params": {}, - "doc": { - "query": "What is the sum of 2 and 2?", - "specific": { - "golds_no_preprocessing": ["4", "four"] - }, - "choices": ["4", "four"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["4"] - }, - "expected_output": { - "em": 1.0, - "f1": 1.0 - }, - "tolerance": 0.01, - "description": "Test DROP with partial match" - } - ] -} diff --git a/tests/unit/metrics/test_cases/exact_match.json b/tests/unit/metrics/test_cases/exact_match.json deleted file mode 100644 index f19b5b2e0..000000000 --- a/tests/unit/metrics/test_cases/exact_match.json +++ /dev/null @@ -1,48 +0,0 @@ -{ - "name": "Exact Match Test Suite", - "description": "Test cases for exact match metric", - "test_cases": [ - { - "name": "Exact Match - Perfect Match", - "metric_class": "exact_match", - "metric_params": {}, - "doc": { - "query": "What is the capital of France?", - "choices": ["Paris", "London", "Berlin"], - "gold_index": 0, - "task_name": "test" - }, - "model_response": { - "text": ["Paris"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "em": 1.0 - }, - "tolerance": 0.01, - "description": "Test exact match with perfect prediction" - }, - { - "name": "Exact Match - No Match", - "metric_class": "exact_match", - "metric_params": {}, - "doc": { - "query": "What is the capital of France?", - "choices": ["Paris", "London", "Berlin"], - "gold_index": 0, - "task_name": "test" - }, - "model_response": { - "text": ["London"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "em": 0.0 - }, - "tolerance": 0.01, - "description": "Test exact match with wrong prediction" - } - ] -} diff --git a/tests/unit/metrics/test_cases/expr_gold_metric.json b/tests/unit/metrics/test_cases/expr_gold_metric.json deleted file mode 100644 index c58c1e900..000000000 --- a/tests/unit/metrics/test_cases/expr_gold_metric.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "name": "Expr Gold Metric Test Suite", - "description": "Test cases for expr_gold_metric metric", - "test_cases": [ - { - "name": "Expr Gold Metric - Basic Test", - "metric_class": "expr_gold_metric", - "metric_params": {}, - "doc": { - "query": "Test query for expr_gold_metric", - "choices": [ - "Test choice 1", - "Test choice 2", - "Test choice 3" - ], - "gold_index": 0, - "task_name": "test" - }, - "model_response": { - "text": [ - "Test choice 1" - ], - "logprobs": [ - 0.5, - 0.3, - 0.2 - ], - "output_tokens": [ - [ - 1 - ], - [ - 2 - ], - [ - 3 - ] - ] - }, - "expected_output": { - "extractive_match": 1.0 - }, - "tolerance": 0.01, - "description": "Basic test case for expr_gold_metric metric" - } - ] -} diff --git a/tests/unit/metrics/test_cases/extractiveness.json b/tests/unit/metrics/test_cases/extractiveness.json deleted file mode 100644 index 1b8178239..000000000 --- a/tests/unit/metrics/test_cases/extractiveness.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "name": "Extractiveness Test Suite", - "description": "Test cases for extractiveness metric", - "test_cases": [ - { - "name": "Extractiveness - High Extractiveness", - "metric_class": "extractiveness", - "metric_params": {}, - "doc": { - "specific": { - "text": "The quick brown fox jumps over the lazy dog. The fox is very fast and agile." - }, - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["The quick brown fox jumps over the lazy dog"] - }, - "expected_output": { - "summarization_coverage": 1.0, - "summarization_density": 9.0, - "summarization_compression": 2.0 - }, - "tolerance": 0.1, - "description": "Test extractiveness with partial extraction" - }, - { - "name": "Extractiveness - Low Extractiveness", - "metric_class": "extractiveness", - "metric_params": {}, - "doc": { - "specific": { - "text": "The quick brown fox jumps over the lazy dog" - }, - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["A fox jumps"] - }, - "expected_output": { - "summarization_coverage": 0.6666666666666666, - "summarization_density": 1.3333333333333333, - "summarization_compression": 3.0 - }, - "tolerance": 0.1, - "description": "Test extractiveness with low extraction" - }, - { - "name": "Extractiveness - Perfect Extraction", - "metric_class": "extractiveness", - "metric_params": {}, - "doc": { - "specific": { - "text": "The quick brown fox jumps over the lazy dog" - }, - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["The quick brown fox jumps over the lazy dog"] - }, - "expected_output": { - "summarization_coverage": 1.0, - "summarization_density": 9.0, - "summarization_compression": 1.0 - }, - "tolerance": 0.01, - "description": "Test extractiveness with perfect extraction" - } - ] -} diff --git a/tests/unit/metrics/test_cases/f1_score.json b/tests/unit/metrics/test_cases/f1_score.json deleted file mode 100644 index e62ff8fb2..000000000 --- a/tests/unit/metrics/test_cases/f1_score.json +++ /dev/null @@ -1,153 +0,0 @@ -{ - "name": "F1 Score Test Suite", - "description": "Test cases for F1 score metric", - "test_cases": [ - { - "name": "F1 Score - Perfect Match", - "metric_class": "f1_score", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["The quick brown fox jumps over the lazy dog"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "f1": 1.0 - }, - "tolerance": 0.01, - "description": "Test F1 score with perfect match" - }, - { - "name": "F1 Score - Partial Match", - "metric_class": "f1_score", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["The quick brown fox"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "f1": 0.6153846153846154 - }, - "tolerance": 0.1, - "description": "Test F1 score with partial match" - }, - { - "name": "F1 Score - No Match", - "metric_class": "f1_score", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["A cat sleeps on the mat"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "f1": 0.13333333333333333 - }, - "tolerance": 0.01, - "description": "Test F1 score with no match" - }, - { - "name": "F1 Score - Different Word Order", - "metric_class": "f1_score", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["The brown quick fox jumps over the dog lazy"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "f1": 1.0 - }, - "tolerance": 0.01, - "description": "Test F1 score with different word order (bag of words)" - }, - { - "name": "F1 Score - Extra Words", - "metric_class": "f1_score", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["The quick brown fox jumps over the lazy dog and runs fast"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "f1": 0.8 - }, - "tolerance": 0.1, - "description": "Test F1 score with extra words in prediction" - }, - { - "name": "F1 Score - Missing Words", - "metric_class": "f1_score", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["The fox jumps over the dog"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "f1": 0.8 - }, - "tolerance": 0.1, - "description": "Test F1 score with missing words in prediction" - }, - { - "name": "F1 Score - Multiple Gold References", - "metric_class": "f1_score", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog", "A fox jumps over a dog"], - "gold_index": [0, 1], - "task_name": "summarization" - }, - "model_response": { - "text": ["The quick brown fox jumps over the lazy dog"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "f1": 1.0 - }, - "tolerance": 0.01, - "description": "Test F1 score with multiple gold references" - } - ] -} diff --git a/tests/unit/metrics/test_cases/f1_score_macro.json b/tests/unit/metrics/test_cases/f1_score_macro.json deleted file mode 100644 index 5a7f32eac..000000000 --- a/tests/unit/metrics/test_cases/f1_score_macro.json +++ /dev/null @@ -1,167 +0,0 @@ -{ - "name": "F1 Score Macro Test Suite", - "description": "Test cases for f1_score_macro metric (corpus-level macro F1 score)", - "corpus_level": true, - "test_cases": [ - { - "name": "F1 Score Macro - Perfect Predictions", - "metric_class": "f1_score_macro", - "metric_params": {}, - "metric_name": "f1", - "docs": [ - { - "query": "What is the capital of France?", - "choices": ["Paris", "London", "Berlin"], - "gold_index": 0, - "task_name": "geography" - }, - { - "query": "What is 2 + 2?", - "choices": ["3", "4", "5"], - "gold_index": 1, - "task_name": "math" - }, - { - "query": "What color is the sky?", - "choices": ["Red", "Blue", "Green"], - "gold_index": 1, - "task_name": "science" - } - ], - "model_responses": [ - { - "text": ["Paris"] - }, - { - "text": ["4"] - }, - { - "text": ["Blue"] - } - ], - "expected_output": 1.0, - "tolerance": 0.01, - "description": "Perfect predictions - all model outputs exactly match the gold choices" - }, - { - "name": "F1 Score Macro - Balanced Performance", - "metric_class": "f1_score_macro", - "metric_params": {}, - "metric_name": "f1", - "docs": [ - { - "query": "Summarize: The quick brown fox jumps over the lazy dog", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - { - "query": "What is the weather like?", - "choices": ["It is sunny and warm today"], - "gold_index": 0, - "task_name": "weather" - }, - { - "query": "Describe a cat", - "choices": ["A cat is a small furry animal"], - "gold_index": 0, - "task_name": "description" - } - ], - "model_responses": [ - { - "text": ["The quick brown fox"] - }, - { - "text": ["It is sunny today"] - }, - { - "text": ["A cat is furry"] - } - ], - "expected_output": 0.0, - "tolerance": 0.1, - "description": "Balanced partial matches - all samples have similar word overlap levels" - }, - { - "name": "F1 Score Macro - Mixed Performance", - "metric_class": "f1_score_macro", - "metric_params": {}, - "metric_name": "f1", - "docs": [ - { - "query": "What is the capital of Japan?", - "choices": ["Tokyo"], - "gold_index": 0, - "task_name": "geography" - }, - { - "query": "What is 5 x 5?", - "choices": ["25"], - "gold_index": 0, - "task_name": "math" - }, - { - "query": "What is the largest planet?", - "choices": ["Jupiter"], - "gold_index": 0, - "task_name": "science" - } - ], - "model_responses": [ - { - "text": ["Tokyo"] - }, - { - "text": ["30"] - }, - { - "text": ["Jupiter"] - } - ], - "expected_output": 0.5, - "tolerance": 0.1, - "description": "Mixed performance - 2 perfect matches, 1 no match (macro average of individual F1s)" - }, - { - "name": "F1 Score Macro - No Matches", - "metric_class": "f1_score_macro", - "metric_params": {}, - "metric_name": "f1", - "docs": [ - { - "query": "What is the main ingredient in pizza?", - "choices": ["Cheese is the main ingredient in pizza"], - "gold_index": 0, - "task_name": "cooking" - }, - { - "query": "What is the opposite of hot?", - "choices": ["Cold"], - "gold_index": 0, - "task_name": "vocabulary" - }, - { - "query": "What is the largest ocean?", - "choices": ["The Pacific Ocean is the largest"], - "gold_index": 0, - "task_name": "geography" - } - ], - "model_responses": [ - { - "text": ["Tomato sauce"] - }, - { - "text": ["Warm"] - }, - { - "text": ["Atlantic Ocean"] - } - ], - "expected_output": 0.0, - "tolerance": 0.01, - "description": "No matches - all model outputs have zero word overlap with gold choices" - } - ] -} diff --git a/tests/unit/metrics/test_cases/f1_score_micro.json b/tests/unit/metrics/test_cases/f1_score_micro.json deleted file mode 100644 index fec84f793..000000000 --- a/tests/unit/metrics/test_cases/f1_score_micro.json +++ /dev/null @@ -1,167 +0,0 @@ -{ - "name": "F1 Score Micro Test Suite", - "description": "Test cases for f1_score_micro metric (corpus-level micro F1 score)", - "corpus_level": true, - "test_cases": [ - { - "name": "F1 Score Micro - Perfect Predictions", - "metric_class": "f1_score_micro", - "metric_name": "f1", - "metric_params": {}, - "docs": [ - { - "query": "What is the capital of France?", - "choices": ["Paris", "London", "Berlin"], - "gold_index": 0, - "task_name": "geography" - }, - { - "query": "What is 2 + 2?", - "choices": ["3", "4", "5"], - "gold_index": 1, - "task_name": "math" - }, - { - "query": "What color is the sky?", - "choices": ["Red", "Blue", "Green"], - "gold_index": 1, - "task_name": "science" - } - ], - "model_responses": [ - { - "text": ["Paris"] - }, - { - "text": ["4"] - }, - { - "text": ["Blue"] - } - ], - "expected_output": 1.0, - "tolerance": 0.01, - "description": "Perfect predictions - all model outputs exactly match the gold choices" - }, - { - "name": "F1 Score Micro - Partial Matches", - "metric_class": "f1_score_micro", - "metric_name": "f1", - "metric_params": {}, - "docs": [ - { - "query": "Summarize: The quick brown fox jumps over the lazy dog", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - { - "query": "What is the weather like?", - "choices": ["It is sunny and warm today"], - "gold_index": 0, - "task_name": "weather" - }, - { - "query": "Describe a cat", - "choices": ["A cat is a small furry animal"], - "gold_index": 0, - "task_name": "description" - } - ], - "model_responses": [ - { - "text": ["The quick brown fox"] - }, - { - "text": ["It is sunny today"] - }, - { - "text": ["A cat is furry"] - } - ], - "expected_output": 0.0, - "tolerance": 0.1, - "description": "Partial matches - model outputs contain some but not all words from gold choices" - }, - { - "name": "F1 Score Micro - No Matches", - "metric_class": "f1_score_micro", - "metric_name": "f1", - "metric_params": {}, - "docs": [ - { - "query": "What is the capital of Japan?", - "choices": ["Tokyo"], - "gold_index": 0, - "task_name": "geography" - }, - { - "query": "What is 5 x 5?", - "choices": ["25"], - "gold_index": 0, - "task_name": "math" - }, - { - "query": "What is the largest planet?", - "choices": ["Jupiter"], - "gold_index": 0, - "task_name": "science" - } - ], - "model_responses": [ - { - "text": ["London"] - }, - { - "text": ["30"] - }, - { - "text": ["Mars"] - } - ], - "expected_output": 0.0, - "tolerance": 0.01, - "description": "No matches - model outputs have no word overlap with gold choices" - }, - { - "name": "F1 Score Micro - Mixed Performance", - "metric_class": "f1_score_micro", - "metric_params": {}, - "metric_name": "f1", - "docs": [ - { - "query": "What is the main ingredient in pizza?", - "choices": ["Cheese is the main ingredient in pizza"], - "gold_index": 0, - "task_name": "cooking" - }, - { - "query": "What is the opposite of hot?", - "choices": ["Cold"], - "gold_index": 0, - "task_name": "vocabulary" - }, - { - "query": "What is the largest ocean?", - "choices": ["The Pacific Ocean is the largest"], - "gold_index": 0, - "task_name": "geography" - } - ], - "model_responses": [ - { - "text": ["Cheese is the main ingredient"] - }, - { - "text": ["Hot"] - }, - { - "text": ["The Pacific Ocean"] - } - ], - "expected_output": 0.0, - "tolerance": 0.1, - "description": "Mixed performance - one perfect match, one no match, one partial match" - } - ] -} diff --git a/tests/unit/metrics/test_cases/faithfulness.json b/tests/unit/metrics/test_cases/faithfulness.json deleted file mode 100644 index 24827b7e4..000000000 --- a/tests/unit/metrics/test_cases/faithfulness.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "name": "Faithfulness Test Suite", - "description": "Test cases for faithfulness metric", - "test_cases": [ - { - "name": "Faithfulness - Basic Test", - "metric_class": "faithfulness", - "metric_params": {}, - "doc": { - "specific": { - "text": "Test query for faithfulness" - }, - "query": "Test query for faithfulness", - "choices": [ - "Test choice 1", - "Test choice 2", - "Test choice 3" - ], - "gold_index": 0, - "task_name": "test" - }, - "model_response": { - "text": [ - "Test choice 1" - ] - }, - "expected_output": { - "summac": -0.516 - }, - "tolerance": 0.01, - "description": "Basic test case for faithfulness metric" - }, - { - "name": "Faithfulness - High Faithfulness Test", - "metric_class": "faithfulness", - "metric_params": {}, - "doc": { - "specific": { - "text": "The quick brown fox jumps over the lazy dog. This sentence contains all the letters of the English alphabet. It is commonly used for testing typing skills and font displays." - }, - "query": "Summarize the text about the fox and dog", - "choices": [ - "A fox jumps over a dog", - "The quick brown fox jumps over the lazy dog", - "A sentence with all alphabet letters" - ], - "gold_index": 1, - "task_name": "summarization" - }, - "model_response": { - "text": [ - "The quick brown fox jumps over the lazy dog. This sentence contains all the letters of the English alphabet." - ] - }, - "expected_output": { - "summac": 0.20 - }, - "tolerance": 0.01, - "description": "Test case with high faithfulness - model output closely matches source text" - }, - { - "name": "Faithfulness - Low Faithfulness Test", - "metric_class": "faithfulness", - "metric_params": {}, - "doc": { - "specific": { - "text": "The weather today is sunny with clear skies. Temperature is expected to reach 25 degrees Celsius. There is no chance of rain according to the forecast." - }, - "query": "What's the weather like?", - "choices": [ - "It's sunny and warm", - "It's raining heavily", - "The weather is unclear" - ], - "gold_index": 0, - "task_name": "weather_qa" - }, - "model_response": { - "text": [ - "It's raining heavily with thunderstorms expected throughout the day. The temperature will drop to 10 degrees and there's a 90% chance of precipitation." - ] - }, - "expected_output": { - "summac": -0.997 - }, - "tolerance": 0.01, - "description": "Test case with low faithfulness - model output contradicts source text" - } - ] -} diff --git a/tests/unit/metrics/test_cases/g_pass_at_k.json b/tests/unit/metrics/test_cases/g_pass_at_k.json deleted file mode 100644 index 8f016c8fb..000000000 --- a/tests/unit/metrics/test_cases/g_pass_at_k.json +++ /dev/null @@ -1,316 +0,0 @@ -{ - "name": "G Pass At K Test Suite", - "description": "Comprehensive test cases for g_pass_at_k metric covering various scenarios including multiple samples, different k values, thresholds, and general text content", - "test_cases": [ - { - "name": "G Pass At K - Basic Single Sample Correct", - "metric_class": "g_pass_at_k", - "metric_params": { - "k": 1, - "n": 1, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is the capital of France?", - "choices": ["Paris", "London", "Berlin"], - "gold_index": 0, - "task_name": "geography" - }, - "model_response": { - "text": ["Paris"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "g-pass@1_0.0": 1.0, - "g-pass@1_0.25": 1.0, - "g-pass@1_0.5": 1.0, - "g-pass@1_0.75": 1.0, - "g-pass@1_1.0": 1.0, - "mg-pass@1": 0.0 - }, - "tolerance": 0.01, - "description": "Basic test case with single correct sample" - }, - { - "name": "G Pass At K - Multiple Samples All Correct", - "metric_class": "g_pass_at_k", - "metric_params": { - "k": 2, - "n": 3, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is the largest planet in our solar system?", - "choices": ["Jupiter"], - "gold_index": 0, - "task_name": "astronomy" - }, - "model_response": { - "text": ["Jupiter", "Jupiter", "Jupiter"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "g-pass@2_0.0": 1.0, - "g-pass@2_0.25": 1.0, - "g-pass@2_0.5": 1.0, - "g-pass@2_0.75": 1.0, - "g-pass@2_1.0": 1.0, - "mg-pass@2": 1.0 - }, - "tolerance": 0.01, - "description": "Test case with multiple samples all correct" - }, - { - "name": "G Pass At K - Mixed Correct and Incorrect", - "metric_class": "g_pass_at_k", - "metric_params": { - "k": 2, - "n": 4, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "Who wrote Romeo and Juliet?", - "choices": ["William Shakespeare"], - "gold_index": 0, - "task_name": "literature" - }, - "model_response": { - "text": ["William Shakespeare", "Shakespeare", "William Shakespeare", "Charles Dickens"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "g-pass@2_0.0": 0.8333333333333334, - "g-pass@2_0.25": 0.8333333333333334, - "g-pass@2_0.5": 0.8333333333333334, - "g-pass@2_0.75": 0.16666666666666666, - "g-pass@2_1.0": 0.16666666666666666, - "mg-pass@2": 0.16666666666666666 - }, - "tolerance": 0.01, - "description": "Test case with mixed correct and incorrect samples" - }, - { - "name": "G Pass At K - Case Sensitivity", - "metric_class": "g_pass_at_k", - "metric_params": { - "k": 1, - "n": 2, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is the chemical symbol for gold?", - "choices": ["Au"], - "gold_index": 0, - "task_name": "chemistry" - }, - "model_response": { - "text": ["Au", "au"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "g-pass@1_0.0": 0.5, - "g-pass@1_0.25": 0.5, - "g-pass@1_0.5": 0.5, - "g-pass@1_0.75": 0.5, - "g-pass@1_1.0": 0.5, - "mg-pass@1": 0.0 - }, - "tolerance": 0.01, - "description": "Test case with case sensitivity (strip_strings should handle this)" - }, - { - "name": "G Pass At K - All Incorrect Samples", - "metric_class": "g_pass_at_k", - "metric_params": { - "k": 1, - "n": 3, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What year did World War II end?", - "choices": ["1945"], - "gold_index": 0, - "task_name": "history" - }, - "model_response": { - "text": ["1944", "1946", "1939"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "g-pass@1_0.0": 0.0, - "g-pass@1_0.25": 0.0, - "g-pass@1_0.5": 0.0, - "g-pass@1_0.75": 0.0, - "g-pass@1_1.0": 0.0, - "mg-pass@1": 0.0 - }, - "tolerance": 0.01, - "description": "Test case with all incorrect samples" - }, - { - "name": "G Pass At K - High K Value", - "metric_class": "g_pass_at_k", - "metric_params": { - "k": 5, - "n": 8, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is the speed of light in vacuum?", - "choices": ["299,792,458 meters per second"], - "gold_index": 0, - "task_name": "physics" - }, - "model_response": { - "text": ["299,792,458 meters per second", "3x10^8 m/s", "299,792,458 meters per second", "300,000 km/s", "299,792,458 meters per second", "c", "299,792,458 meters per second", "186,282 miles per second"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "g-pass@5_0.0": 1.0, - "g-pass@5_0.25": 0.9285714285714286, - "g-pass@5_0.5": 0.5, - "g-pass@5_0.75": 0.07142857142857142, - "g-pass@5_1.0": 0.0, - "mg-pass@5": 0.02857142857142857 - }, - "tolerance": 0.01, - "description": "Test case with high k value and multiple correct samples" - }, - { - "name": "G Pass At K - Long Text Answer", - "metric_class": "g_pass_at_k", - "metric_params": { - "k": 1, - "n": 2, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is the main theme of George Orwell's 1984?", - "choices": ["Totalitarianism and surveillance"], - "gold_index": 0, - "task_name": "literature" - }, - "model_response": { - "text": ["Totalitarianism and surveillance", "Dystopian society"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "g-pass@1_0.0": 0.5, - "g-pass@1_0.25": 0.5, - "g-pass@1_0.5": 0.5, - "g-pass@1_0.75": 0.5, - "g-pass@1_1.0": 0.5, - "mg-pass@1": 0.0 - }, - "tolerance": 0.01, - "description": "Test case with longer text answers" - }, - { - "name": "G Pass At K - Numeric Answer", - "metric_class": "g_pass_at_k", - "metric_params": { - "k": 1, - "n": 3, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "How many sides does a hexagon have?", - "choices": ["6"], - "gold_index": 0, - "task_name": "geometry" - }, - "model_response": { - "text": ["6", "six", "Six"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "g-pass@1_0.0": 0.3333333333333333, - "g-pass@1_0.25": 0.3333333333333333, - "g-pass@1_0.5": 0.3333333333333333, - "g-pass@1_0.75": 0.3333333333333333, - "g-pass@1_1.0": 0.3333333333333333, - "mg-pass@1": 0.0 - }, - "tolerance": 0.01, - "description": "Test case with numeric answers" - }, - { - "name": "G Pass At K - Partial Match", - "metric_class": "g_pass_at_k", - "metric_params": { - "k": 2, - "n": 4, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is the full name of the author of Pride and Prejudice?", - "choices": ["Jane Austen"], - "gold_index": 0, - "task_name": "literature" - }, - "model_response": { - "text": ["Jane Austen", "Austen", "Jane Austen", "J. Austen"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "g-pass@2_0.0": 0.8333333333333334, - "g-pass@2_0.25": 0.8333333333333334, - "g-pass@2_0.5": 0.8333333333333334, - "g-pass@2_0.75": 0.16666666666666666, - "g-pass@2_1.0": 0.16666666666666666, - "mg-pass@2": 0.16666666666666666 - }, - "tolerance": 0.01, - "description": "Test case with partial matches (exact string matching)" - }, - { - "name": "G Pass At K - Edge Case Empty String", - "metric_class": "g_pass_at_k", - "metric_params": { - "k": 1, - "n": 1, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is the answer to this question?", - "choices": [""], - "gold_index": 0, - "task_name": "test" - }, - "model_response": { - "text": [""], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "g-pass@1_0.0": 1.0, - "g-pass@1_0.25": 1.0, - "g-pass@1_0.5": 1.0, - "g-pass@1_0.75": 1.0, - "g-pass@1_1.0": 1.0, - "mg-pass@1": 0.0 - }, - "tolerance": 0.01, - "description": "Edge case with empty string" - } - ] -} diff --git a/tests/unit/metrics/test_cases/g_pass_at_k_latex.json b/tests/unit/metrics/test_cases/g_pass_at_k_latex.json deleted file mode 100644 index afd7580de..000000000 --- a/tests/unit/metrics/test_cases/g_pass_at_k_latex.json +++ /dev/null @@ -1,223 +0,0 @@ -{ - "name": "G Pass At K Latex Test Suite", - "description": "Comprehensive test cases for g_pass_at_k_latex metric covering various scenarios including multiple samples, different k values, thresholds, and mathematical content", - "test_cases": [ - { - "name": "G Pass At K Latex - Basic Single Sample Correct", - "metric_class": "g_pass_at_k_latex", - "metric_params": { - "k": 1, - "n": 1, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is 2+2?", - "choices": ["$\\frac{1}{2}$"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["$\\frac{1}{2}$"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "latex_g-pass@1_0.0": 1.0, - "latex_g-pass@1_0.25": 1.0, - "latex_g-pass@1_0.5": 1.0, - "latex_g-pass@1_0.75": 1.0, - "latex_g-pass@1_1.0": 1.0, - "mlatex_g-pass@1": 0.0 - }, - "tolerance": 0.01, - "description": "Basic test case with single correct sample" - }, - { - "name": "G Pass At K Latex - Multiple Samples All Correct", - "metric_class": "g_pass_at_k_latex", - "metric_params": { - "k": 2, - "n": 3, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is the derivative of x^2?", - "choices": ["$2x$"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["$2x$", "$2x$", "$2x$"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "latex_g-pass@2_0.0": 1.0, - "latex_g-pass@2_0.25": 1.0, - "latex_g-pass@2_0.5": 1.0, - "latex_g-pass@2_0.75": 1.0, - "latex_g-pass@2_1.0": 1.0, - "mlatex_g-pass@2": 1.0 - }, - "tolerance": 0.01, - "description": "Test case with multiple samples all correct" - }, - { - "name": "G Pass At K Latex - Mixed Correct and Incorrect", - "metric_class": "g_pass_at_k_latex", - "metric_params": { - "k": 2, - "n": 4, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is the integral of x?", - "choices": ["$\\frac{x^2}{2}$"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["$\\frac{x^2}{2}$", "$x$", "$\\frac{x^2}{2}$", "$x^2$"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "latex_g-pass@2_0.0": 0.8333333333333334, - "latex_g-pass@2_0.25": 0.8333333333333334, - "latex_g-pass@2_0.5": 0.8333333333333334, - "latex_g-pass@2_0.75": 0.16666666666666666, - "latex_g-pass@2_1.0": 0.16666666666666666, - "mlatex_g-pass@2": 0.16666666666666666 - }, - "tolerance": 0.01, - "description": "Test case with mixed correct and incorrect samples" - }, - { - "name": "G Pass At K Latex - Complex LaTeX Expression", - "metric_class": "g_pass_at_k_latex", - "metric_params": { - "k": 1, - "n": 2, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is the quadratic formula?", - "choices": ["$x = \\frac{-b \\pm \\sqrt{b^2 - 4ac}}{2a}$"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["$x = \\frac{-b \\pm \\sqrt{b^2 - 4ac}}{2a}$", "$x = \\frac{-b + \\sqrt{b^2 - 4ac}}{2a}$"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "latex_g-pass@1_0.0": 0.5, - "latex_g-pass@1_0.25": 0.5, - "latex_g-pass@1_0.5": 0.5, - "latex_g-pass@1_0.75": 0.5, - "latex_g-pass@1_1.0": 0.5, - "mlatex_g-pass@1": 0.0 - }, - "tolerance": 0.01, - "description": "Test case with complex LaTeX expression" - }, - { - "name": "G Pass At K Latex - All Incorrect Samples", - "metric_class": "g_pass_at_k_latex", - "metric_params": { - "k": 1, - "n": 3, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is the limit of 1/x as x approaches infinity?", - "choices": ["$0$"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["$1$", "$\\infty$", "$\\text{undefined}$"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "latex_g-pass@1_0.0": 0.0, - "latex_g-pass@1_0.25": 0.0, - "latex_g-pass@1_0.5": 0.0, - "latex_g-pass@1_0.75": 0.0, - "latex_g-pass@1_1.0": 0.0, - "mlatex_g-pass@1": 0.0 - }, - "tolerance": 0.01, - "description": "Test case with all incorrect samples" - }, - { - "name": "G Pass At K Latex - High K Value", - "metric_class": "g_pass_at_k_latex", - "metric_params": { - "k": 5, - "n": 8, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is the sum of the first n natural numbers?", - "choices": ["$\\frac{n(n+1)}{2}$"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["$\\frac{n(n+1)}{2}$", "$n(n+1)/2$", "$\\frac{n(n+1)}{2}$", "$n^2/2$", "$\\frac{n(n+1)}{2}$", "$n+1$", "$\\frac{n(n+1)}{2}$", "$n$"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "latex_g-pass@5_0.0": 1.0, - "latex_g-pass@5_0.25": 1.0, - "latex_g-pass@5_0.5": 0.8214285714285715, - "latex_g-pass@5_0.75": 0.28571428571428564, - "latex_g-pass@5_1.0": 0.017857142857142853, - "mlatex_g-pass@5": 0.1214285714285714 - }, - "tolerance": 0.01, - "description": "Test case with high k value and multiple correct samples" - }, - { - "name": "G Pass At K Latex - Edge Case Single Sample", - "metric_class": "g_pass_at_k_latex", - "metric_params": { - "k": 1, - "n": 1, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is the value of pi?", - "choices": ["$\\pi$"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["$3.14159$"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "latex_g-pass@1_0.0": 0.0, - "latex_g-pass@1_0.25": 0.0, - "latex_g-pass@1_0.5": 0.0, - "latex_g-pass@1_0.75": 0.0, - "latex_g-pass@1_1.0": 0.0, - "mlatex_g-pass@1": 0.0 - }, - "tolerance": 0.01, - "description": "Edge case with single incorrect sample" - } - ] -} diff --git a/tests/unit/metrics/test_cases/g_pass_at_k_math.json b/tests/unit/metrics/test_cases/g_pass_at_k_math.json deleted file mode 100644 index 0bd2f20e3..000000000 --- a/tests/unit/metrics/test_cases/g_pass_at_k_math.json +++ /dev/null @@ -1,347 +0,0 @@ -{ - "name": "G Pass At K Math Test Suite", - "description": "Comprehensive test cases for g_pass_at_k_math metric covering various scenarios including multiple samples, different k values, thresholds, and mathematical content", - "test_cases": [ - { - "name": "G Pass At K Math - Basic Single Sample Correct", - "metric_class": "g_pass_at_k_math", - "metric_params": { - "k": 1, - "n": 1, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is 2+2?", - "choices": ["4", "5", "6"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["4"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "math_g-pass@1_0.0": 1.0, - "math_g-pass@1_0.25": 1.0, - "math_g-pass@1_0.5": 1.0, - "math_g-pass@1_0.75": 1.0, - "math_g-pass@1_1.0": 1.0, - "mmath_g-pass@1": 0.0 - }, - "tolerance": 0.01, - "description": "Basic test case with single correct sample" - }, - { - "name": "G Pass At K Math - Multiple Samples All Correct", - "metric_class": "g_pass_at_k_math", - "metric_params": { - "k": 2, - "n": 3, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is the derivative of x^2?", - "choices": ["2x"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["2x", "2x", "2x"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "math_g-pass@2_0.0": 1.0, - "math_g-pass@2_0.25": 1.0, - "math_g-pass@2_0.5": 1.0, - "math_g-pass@2_0.75": 1.0, - "math_g-pass@2_1.0": 1.0, - "mmath_g-pass@2": 1.0 - }, - "tolerance": 0.01, - "description": "Test case with multiple samples all correct" - }, - { - "name": "G Pass At K Math - Mixed Correct and Incorrect", - "metric_class": "g_pass_at_k_math", - "metric_params": { - "k": 2, - "n": 4, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is the integral of x?", - "choices": ["x^2/2"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["x^2/2", "x", "x^2/2", "x^2"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "math_g-pass@2_0.0": 0.0, - "math_g-pass@2_0.25": 0.0, - "math_g-pass@2_0.5": 0.0, - "math_g-pass@2_0.75": 0.0, - "math_g-pass@2_1.0": 0.0, - "mmath_g-pass@2": 0.0 - }, - "tolerance": 0.01, - "description": "Test case with mixed correct and incorrect samples" - }, - { - "name": "G Pass At K Math - Decimal Numbers", - "metric_class": "g_pass_at_k_math", - "metric_params": { - "k": 1, - "n": 2, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is pi to 2 decimal places?", - "choices": ["3.14"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["3.14", "3.14159"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "math_g-pass@1_0.0": 0.5, - "math_g-pass@1_0.25": 0.5, - "math_g-pass@1_0.5": 0.5, - "math_g-pass@1_0.75": 0.5, - "math_g-pass@1_1.0": 0.5, - "mmath_g-pass@1": 0.0 - }, - "tolerance": 0.01, - "description": "Test case with decimal numbers" - }, - { - "name": "G Pass At K Math - Fractions", - "metric_class": "g_pass_at_k_math", - "metric_params": { - "k": 1, - "n": 3, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is 1/2 + 1/4?", - "choices": ["3/4"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["3/4", "0.75", "1/2"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "math_g-pass@1_0.0": 0.6666666666666667, - "math_g-pass@1_0.25": 0.6666666666666667, - "math_g-pass@1_0.5": 0.6666666666666667, - "math_g-pass@1_0.75": 0.6666666666666667, - "math_g-pass@1_1.0": 0.6666666666666667, - "mmath_g-pass@1": 0.0 - }, - "tolerance": 0.01, - "description": "Test case with fractions" - }, - { - "name": "G Pass At K Math - All Incorrect Samples", - "metric_class": "g_pass_at_k_math", - "metric_params": { - "k": 1, - "n": 3, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is the limit of 1/x as x approaches infinity?", - "choices": ["0"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["1", "infinity", "undefined"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "math_g-pass@1_0.0": 0.0, - "math_g-pass@1_0.25": 0.0, - "math_g-pass@1_0.5": 0.0, - "math_g-pass@1_0.75": 0.0, - "math_g-pass@1_1.0": 0.0, - "mmath_g-pass@1": 0.0 - }, - "tolerance": 0.01, - "description": "Test case with all incorrect samples" - }, - { - "name": "G Pass At K Math - High K Value", - "metric_class": "g_pass_at_k_math", - "metric_params": { - "k": 5, - "n": 8, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is the sum of the first n natural numbers?", - "choices": ["n(n+1)/2"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["n(n+1)/2", "n*(n+1)/2", "n(n+1)/2", "n^2/2", "n(n+1)/2", "n+1", "n(n+1)/2", "n"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "math_g-pass@5_0.0": 0.0, - "math_g-pass@5_0.25": 0.0, - "math_g-pass@5_0.5": 0.0, - "math_g-pass@5_0.75": 0.0, - "math_g-pass@5_1.0": 0.0, - "mmath_g-pass@5": 0.0 - }, - "tolerance": 0.01, - "description": "Test case with high k value and multiple correct samples" - }, - { - "name": "G Pass At K Math - Negative Numbers", - "metric_class": "g_pass_at_k_math", - "metric_params": { - "k": 1, - "n": 2, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is -5 + 3?", - "choices": ["-2"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["-2", "2"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "math_g-pass@1_0.0": 0.5, - "math_g-pass@1_0.25": 0.5, - "math_g-pass@1_0.5": 0.5, - "math_g-pass@1_0.75": 0.5, - "math_g-pass@1_1.0": 0.5, - "mmath_g-pass@1": 0.0 - }, - "tolerance": 0.01, - "description": "Test case with negative numbers" - }, - { - "name": "G Pass At K Math - Complex Expression", - "metric_class": "g_pass_at_k_math", - "metric_params": { - "k": 2, - "n": 4, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is (2+3)*4?", - "choices": ["20"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["20", "24", "20", "14"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "math_g-pass@2_0.0": 0.8333333333333334, - "math_g-pass@2_0.25": 0.8333333333333334, - "math_g-pass@2_0.5": 0.8333333333333334, - "math_g-pass@2_0.75": 0.16666666666666666, - "math_g-pass@2_1.0": 0.16666666666666666, - "mmath_g-pass@2": 0.16666666666666666 - }, - "tolerance": 0.01, - "description": "Test case with complex mathematical expression" - }, - { - "name": "G Pass At K Math - Percentage", - "metric_class": "g_pass_at_k_math", - "metric_params": { - "k": 1, - "n": 2, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is 25% of 80?", - "choices": ["20"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["20", "25"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "math_g-pass@1_0.0": 0.5, - "math_g-pass@1_0.25": 0.5, - "math_g-pass@1_0.5": 0.5, - "math_g-pass@1_0.75": 0.5, - "math_g-pass@1_1.0": 0.5, - "mmath_g-pass@1": 0.0 - }, - "tolerance": 0.01, - "description": "Test case with percentage calculation" - }, - { - "name": "G Pass At K Math - Edge Case Zero", - "metric_class": "g_pass_at_k_math", - "metric_params": { - "k": 1, - "n": 1, - "thresholds": [0.0, 0.25, 0.5, 0.75, 1.0], - "strip_strings": true - }, - "doc": { - "query": "What is 5 - 5?", - "choices": ["0"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["0"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "math_g-pass@1_0.0": 1.0, - "math_g-pass@1_0.25": 1.0, - "math_g-pass@1_0.5": 1.0, - "math_g-pass@1_0.75": 1.0, - "math_g-pass@1_1.0": 1.0, - "mmath_g-pass@1": 0.0 - }, - "tolerance": 0.01, - "description": "Edge case with zero result" - } - ] -} diff --git a/tests/unit/metrics/test_cases/gpqa_instruct_metric.json b/tests/unit/metrics/test_cases/gpqa_instruct_metric.json deleted file mode 100644 index 4dddf83cc..000000000 --- a/tests/unit/metrics/test_cases/gpqa_instruct_metric.json +++ /dev/null @@ -1,447 +0,0 @@ -{ - "name": "Gpqa Instruct Metric Test Suite", - "description": "Test cases for gpqa_instruct_metric metric - tests multiple choice answer extraction (A, B, C, D)", - "test_cases": [ - { - "name": "Basic Answer Extraction - Direct Answer", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nWhat is the capital of France?\n\nA) London\nB) Paris\nC) Berlin\nD) Madrid", - "choices": ["A", "B", "C", "D"], - "gold_index": 1, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "Let me think about this step by step. France is a country in Europe, and its capital city is Paris. This is a well-known fact in geography.\n\nAnswer: B" - ] - }, - "expected_output": { - "extractive_match": 1.0 - }, - "tolerance": 0.01, - "description": "Basic test case with direct answer format" - }, - { - "name": "Answer with Reasoning - Correct Format", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nWhich planet is closest to the Sun?\n\nA) Earth\nB) Venus\nC) Mercury\nD) Mars", - "choices": ["A", "B", "C", "D"], - "gold_index": 2, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "Let me think about this step by step. The planets in order from the Sun are: Mercury, Venus, Earth, Mars, Jupiter, Saturn, Uranus, Neptune. So Mercury is the closest planet to the Sun.\n\nAnswer: C" - ] - }, - "expected_output": { - "extractive_match": 1.0 - }, - "tolerance": 0.01, - "description": "Answer with reasoning but correct final format" - }, - { - "name": "Answer Embedded in Reasoning", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nWhat is 2 + 2?\n\nA) 3\nB) 4\nC) 5\nD) 6", - "choices": ["A", "B", "C", "D"], - "gold_index": 1, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "Let me think about this step by step. 2 + 2 = 4, so the answer is B. This is basic arithmetic.\n\nAnswer: B" - ] - }, - "expected_output": { - "extractive_match": 1.0 - }, - "tolerance": 0.01, - "description": "Answer mentioned in reasoning and correctly formatted at end" - }, - { - "name": "Wrong Answer Format", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nWhat color is the sky?\n\nA) Red\nB) Green\nC) Blue\nD) Yellow", - "choices": ["A", "B", "C", "D"], - "gold_index": 2, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "The sky appears blue due to Rayleigh scattering of sunlight. The answer is C.\n\nAnswer: A" - ] - }, - "expected_output": { - "extractive_match": 0.0 - }, - "tolerance": 0.01, - "description": "Correct reasoning but wrong final answer" - }, - { - "name": "No Answer Format", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nWhat is the largest ocean?\n\nA) Atlantic\nB) Indian\nC) Pacific\nD) Arctic", - "choices": ["A", "B", "C", "D"], - "gold_index": 2, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "The Pacific Ocean is the largest ocean on Earth, covering about one-third of the Earth's surface. It's much larger than the Atlantic, Indian, and Arctic oceans." - ] - }, - "expected_output": { - "extractive_match": 0.0 - }, - "tolerance": 0.01, - "description": "Correct reasoning but no answer format" - }, - { - "name": "Multiple Answer Mentions", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nWhich is a primary color?\n\nA) Orange\nB) Purple\nC) Red\nD) Green", - "choices": ["A", "B", "C", "D"], - "gold_index": 2, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "The primary colors are red, blue, and yellow. So the answer is C (Red). Red is a primary color because it cannot be created by mixing other colors.\n\nAnswer: C" - ] - }, - "expected_output": { - "extractive_match": 1.0 - }, - "tolerance": 0.01, - "description": "Answer mentioned multiple times but correct final format" - }, - { - "name": "Answer with Parentheses", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nWhat is the chemical symbol for gold?\n\nA) Ag\nB) Au\nC) Fe\nD) Cu", - "choices": ["A", "B", "C", "D"], - "gold_index": 1, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "The chemical symbol for gold is Au, which comes from the Latin word 'aurum'. So the answer is (B) Au.\n\nAnswer: B" - ] - }, - "expected_output": { - "extractive_match": 1.0 - }, - "tolerance": 0.01, - "description": "Answer with parentheses but correct final format" - }, - { - "name": "Answer with Period", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nHow many sides does a triangle have?\n\nA) 2\nB) 3\nC) 4\nD) 5", - "choices": ["A", "B", "C", "D"], - "gold_index": 1, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "A triangle is a polygon with three sides. So the answer is B.\n\nAnswer: B." - ] - }, - "expected_output": { - "extractive_match": 1.0 - }, - "tolerance": 0.01, - "description": "Answer with period but still extractable" - }, - { - "name": "Answer with Extra Text", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nWhat is the opposite of hot?\n\nA) Warm\nB) Cold\nC) Lukewarm\nD) Tepid", - "choices": ["A", "B", "C", "D"], - "gold_index": 1, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "The opposite of hot is cold. So the answer is B.\n\nAnswer: B, which is cold." - ] - }, - "expected_output": { - "extractive_match": 1.0 - }, - "tolerance": 0.01, - "description": "Answer with extra text but letter still extractable" - }, - { - "name": "Answer in Different Case", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nWhat is the largest planet?\n\nA) Earth\nB) Mars\nC) Jupiter\nD) Saturn", - "choices": ["A", "B", "C", "D"], - "gold_index": 2, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "Jupiter is the largest planet in our solar system. So the answer is c.\n\nAnswer: c" - ] - }, - "expected_output": { - "extractive_match": 0.0 - }, - "tolerance": 0.01, - "description": "Answer in lowercase but still extractable" - }, - { - "name": "Answer with Reasoning Only", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nWhat is the square root of 16?\n\nA) 2\nB) 4\nC) 8\nD) 16", - "choices": ["A", "B", "C", "D"], - "gold_index": 1, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "The square root of 16 is 4, because 4 × 4 = 16. Therefore, the answer is B." - ] - }, - "expected_output": { - "extractive_match": 1.0 - }, - "tolerance": 0.01, - "description": "Answer embedded in reasoning without formal format" - }, - { - "name": "Answer with Quotes", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nWhat is the capital of Japan?\n\nA) Beijing\nB) Seoul\nC) Tokyo\nD) Bangkok", - "choices": ["A", "B", "C", "D"], - "gold_index": 2, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "The capital of Japan is Tokyo. So the answer is \"C\".\n\nAnswer: \"C\"" - ] - }, - "expected_output": { - "extractive_match": 1.0 - }, - "tolerance": 0.01, - "description": "Answer with quotes but still extractable" - }, - { - "name": "Answer with Multiple Letters", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nWhich of these are mammals?\n\nA) Fish\nB) Birds\nC) Dogs\nD) Reptiles", - "choices": ["A", "B", "C", "D"], - "gold_index": 2, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "Dogs are mammals, while fish, birds, and reptiles are not. So the answer is C.\n\nAnswer: C" - ] - }, - "expected_output": { - "extractive_match": 1.0 - }, - "tolerance": 0.01, - "description": "Multiple letters mentioned but correct single answer" - }, - { - "name": "Answer with Explanation", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nWhat is the chemical formula for water?\n\nA) CO2\nB) H2O\nC) O2\nD) NaCl", - "choices": ["A", "B", "C", "D"], - "gold_index": 1, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "Water is composed of two hydrogen atoms and one oxygen atom, so its chemical formula is H2O. Therefore, the answer is B.\n\nAnswer: B" - ] - }, - "expected_output": { - "extractive_match": 1.0 - }, - "tolerance": 0.01, - "description": "Answer with detailed explanation" - }, - { - "name": "Answer with Confidence", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nWhat is the speed of light?\n\nA) 186,000 mph\nB) 186,000 m/s\nC) 186,000 km/s\nD) 186,000 km/h", - "choices": ["A", "B", "C", "D"], - "gold_index": 2, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "The speed of light in a vacuum is approximately 186,000 miles per second, which is about 300,000 kilometers per second. So the answer is definitely C.\n\nAnswer: C" - ] - }, - "expected_output": { - "extractive_match": 1.0 - }, - "tolerance": 0.01, - "description": "Answer with confidence indicator" - }, - { - "name": "Answer with Step-by-Step", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nWhat is 15 × 3?\n\nA) 35\nB) 40\nC) 45\nD) 50", - "choices": ["A", "B", "C", "D"], - "gold_index": 2, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "Let me solve this step by step:\n1. 15 × 3 = 45\n2. This matches option C\n\nAnswer: C" - ] - }, - "expected_output": { - "extractive_match": 1.0 - }, - "tolerance": 0.01, - "description": "Answer with numbered steps" - }, - { - "name": "Answer with Elimination", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nWhich is not a fruit?\n\nA) Apple\nB) Carrot\nC) Banana\nD) Orange", - "choices": ["A", "B", "C", "D"], - "gold_index": 1, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "Let me eliminate the wrong answers:\n- Apple is a fruit ✓\n- Carrot is a vegetable ✗\n- Banana is a fruit ✓\n- Orange is a fruit ✓\n\nSo the answer is B.\n\nAnswer: B" - ] - }, - "expected_output": { - "extractive_match": 1.0 - }, - "tolerance": 0.01, - "description": "Answer using elimination method" - }, - { - "name": "Answer with Definition", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nWhat is a synonym for 'happy'?\n\nA) Sad\nB) Joyful\nC) Angry\nD) Tired", - "choices": ["A", "B", "C", "D"], - "gold_index": 1, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "A synonym is a word that means the same or nearly the same as another word. 'Happy' means feeling or showing pleasure or contentment. 'Joyful' means feeling, expressing, or causing great pleasure and happiness. Therefore, 'joyful' is a synonym for 'happy', so the answer is B.\n\nAnswer: B" - ] - }, - "expected_output": { - "extractive_match": 1.0 - }, - "tolerance": 0.01, - "description": "Answer with definition and explanation" - }, - { - "name": "Answer with Comparison", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nWhich is larger?\n\nA) 1/2\nB) 1/3\nC) 1/4\nD) 1/5", - "choices": ["A", "B", "C", "D"], - "gold_index": 0, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "Let me compare these fractions:\n1/2 = 0.5\n1/3 ≈ 0.333\n1/4 = 0.25\n1/5 = 0.2\n\n1/2 is the largest, so the answer is A.\n\nAnswer: A" - ] - }, - "expected_output": { - "extractive_match": 1.0 - }, - "tolerance": 0.01, - "description": "Answer with numerical comparison" - }, - { - "name": "Answer with Context", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nIn what year did World War II end?\n\nA) 1943\nB) 1944\nC) 1945\nD) 1946", - "choices": ["A", "B", "C", "D"], - "gold_index": 2, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "World War II ended in 1945. Germany surrendered in May 1945, and Japan surrendered in September 1945 after the atomic bombings of Hiroshima and Nagasaki. So the answer is C.\n\nAnswer: C" - ] - }, - "expected_output": { - "extractive_match": 1.0 - }, - "tolerance": 0.01, - "description": "Answer with historical context" - }, - { - "name": "Answer with Formula", - "metric_class": "gpqa_instruct_metric", - "metric_params": {}, - "doc": { - "query": "Answer the following multiple choice question. The last line of your response should be of the following format: 'Answer: $LETTER' (without quotes) where LETTER is one of ABCD. Think step by step before answering.\n\nWhat is the area of a circle with radius 3?\n\nA) 6π\nB) 9π\nC) 12π\nD) 18π", - "choices": ["A", "B", "C", "D"], - "gold_index": 1, - "task_name": "gpqa_instruct" - }, - "model_response": { - "text": [ - "The area of a circle is given by the formula A = πr².\nFor radius r = 3:\nA = π(3)² = π(9) = 9π\n\nSo the answer is B.\n\nAnswer: B" - ] - }, - "expected_output": { - "extractive_match": 1.0 - }, - "tolerance": 0.01, - "description": "Answer with mathematical formula" - } - ] -} diff --git a/tests/unit/metrics/test_cases/gpqa_instruct_pass_at_k.json b/tests/unit/metrics/test_cases/gpqa_instruct_pass_at_k.json deleted file mode 100644 index 6d2691411..000000000 --- a/tests/unit/metrics/test_cases/gpqa_instruct_pass_at_k.json +++ /dev/null @@ -1,281 +0,0 @@ -{ - "name": "Gpqa Instruct Pass At K Test Suite", - "description": "Comprehensive test cases for gpqa_instruct_pass_at_k metric covering various scenarios including multiple samples, different k values, and multiple choice letter indices (A, B, C, D, etc.)", - "test_cases": [ - { - "name": "Gpqa Instruct Pass At K - Basic Single Sample Correct", - "metric_class": "gpqa_instruct_pass_at_k", - "metric_params": { - "k": 1, - "n": 1, - "strip_strings": true - }, - "doc": { - "query": "What is the capital of France?\nA. London\nB. Paris\nC. Berlin\nD. Madrid", - "choices": ["A", "B", "C", "D"], - "gold_index": 1, - "task_name": "geography" - }, - "model_response": { - "text": ["B"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "gpqa_pass@k:k=1&n=1&strip_strings=True": 1.0 - }, - "tolerance": 0.01, - "description": "Basic test case with single correct sample" - }, - { - "name": "Gpqa Instruct Pass At K - Multiple Samples All Correct", - "metric_class": "gpqa_instruct_pass_at_k", - "metric_params": { - "k": 2, - "n": 3, - "strip_strings": true - }, - "doc": { - "query": "What is the largest planet in our solar system?\nA. Earth\nB. Jupiter\nC. Saturn\nD. Mars", - "choices": ["A", "B", "C", "D"], - "gold_index": 1, - "task_name": "astronomy" - }, - "model_response": { - "text": ["B", "B", "B"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "gpqa_pass@k:k=2&n=3&strip_strings=True": 1.0 - }, - "tolerance": 0.01, - "description": "Test case with multiple samples all correct" - }, - { - "name": "Gpqa Instruct Pass At K - Mixed Correct and Incorrect", - "metric_class": "gpqa_instruct_pass_at_k", - "metric_params": { - "k": 2, - "n": 4, - "strip_strings": true - }, - "doc": { - "query": "Who wrote Romeo and Juliet?\nA. Charles Dickens\nB. William Shakespeare\nC. Jane Austen\nD. Mark Twain", - "choices": ["A", "B", "C", "D"], - "gold_index": 1, - "task_name": "literature" - }, - "model_response": { - "text": ["B", "A", "B", "C"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "gpqa_pass@k:k=2&n=4&strip_strings=True": 0.8333333333333333 - }, - "tolerance": 0.01, - "description": "Test case with mixed correct and incorrect samples" - }, - { - "name": "Gpqa Instruct Pass At K - Case Sensitivity", - "metric_class": "gpqa_instruct_pass_at_k", - "metric_params": { - "k": 1, - "n": 2, - "strip_strings": true - }, - "doc": { - "query": "What is the chemical symbol for gold?\nA. Ag\nB. Au\nC. Fe\nD. Cu", - "choices": ["A", "B", "C", "D"], - "gold_index": 1, - "task_name": "chemistry" - }, - "model_response": { - "text": ["B", "b"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "gpqa_pass@k:k=1&n=2&strip_strings=True": 0.5 - }, - "tolerance": 0.01, - "description": "Test case with case sensitivity (strip_strings should handle this)" - }, - { - "name": "Gpqa Instruct Pass At K - All Incorrect Samples", - "metric_class": "gpqa_instruct_pass_at_k", - "metric_params": { - "k": 1, - "n": 3, - "strip_strings": true - }, - "doc": { - "query": "What year did World War II end?\nA. 1943\nB. 1944\nC. 1945\nD. 1946", - "choices": ["A", "B", "C", "D"], - "gold_index": 2, - "task_name": "history" - }, - "model_response": { - "text": ["A", "B", "D"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "gpqa_pass@k:k=1&n=3&strip_strings=True": 0.0 - }, - "tolerance": 0.01, - "description": "Test case with all incorrect samples" - }, - { - "name": "Gpqa Instruct Pass At K - High K Value", - "metric_class": "gpqa_instruct_pass_at_k", - "metric_params": { - "k": 5, - "n": 8, - "strip_strings": true - }, - "doc": { - "query": "What is the speed of light in vacuum?\nA. 299,792,458 m/s\nB. 300,000 km/s\nC. 186,282 miles/s\nD. 3x10^8 m/s", - "choices": ["A", "B", "C", "D"], - "gold_index": 0, - "task_name": "physics" - }, - "model_response": { - "text": ["A", "B", "A", "C", "A", "D", "A", "B"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "gpqa_pass@k:k=5&n=8&strip_strings=True": 1.0 - }, - "tolerance": 0.01, - "description": "Test case with high k value and multiple correct samples" - }, - { - "name": "Gpqa Instruct Pass At K - Parentheses Format", - "metric_class": "gpqa_instruct_pass_at_k", - "metric_params": { - "k": 1, - "n": 2, - "strip_strings": true - }, - "doc": { - "query": "What is the main theme of George Orwell's 1984?\nA. Love and romance\nB. Totalitarianism and surveillance\nC. War and peace\nD. Economic inequality", - "choices": ["A", "B", "C", "D"], - "gold_index": 1, - "task_name": "literature" - }, - "model_response": { - "text": ["(B)", "B"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "gpqa_pass@k:k=1&n=2&strip_strings=True": 1.0 - }, - "tolerance": 0.01, - "description": "Test case with parentheses format" - }, - { - "name": "Gpqa Instruct Pass At K - Reasoning with Answer", - "metric_class": "gpqa_instruct_pass_at_k", - "metric_params": { - "k": 1, - "n": 2, - "strip_strings": true - }, - "doc": { - "query": "How many sides does a hexagon have?\nA. 4\nB. 5\nC. 6\nD. 7", - "choices": ["A", "B", "C", "D"], - "gold_index": 2, - "task_name": "geometry" - }, - "model_response": { - "text": ["A hexagon has 6 sides, so the answer is C", "C"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "gpqa_pass@k:k=1&n=2&strip_strings=True": 1.0 - }, - "tolerance": 0.01, - "description": "Test case with reasoning and answer extraction" - }, - { - "name": "Gpqa Instruct Pass At K - Final Answer Format", - "metric_class": "gpqa_instruct_pass_at_k", - "metric_params": { - "k": 1, - "n": 2, - "strip_strings": true - }, - "doc": { - "query": "What is the largest ocean on Earth?\nA. Atlantic Ocean\nB. Indian Ocean\nC. Pacific Ocean\nD. Arctic Ocean", - "choices": ["A", "B", "C", "D"], - "gold_index": 2, - "task_name": "geography" - }, - "model_response": { - "text": ["The largest ocean is the Pacific Ocean. Final answer is C", "C"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "gpqa_pass@k:k=1&n=2&strip_strings=True": 1.0 - }, - "tolerance": 0.01, - "description": "Test case with 'final answer' format" - }, - { - "name": "Gpqa Instruct Pass At K - Edge Case Single Choice", - "metric_class": "gpqa_instruct_pass_at_k", - "metric_params": { - "k": 1, - "n": 1, - "strip_strings": true - }, - "doc": { - "query": "Is the Earth round?\nA. Yes", - "choices": ["A"], - "gold_index": 0, - "task_name": "science" - }, - "model_response": { - "text": ["A"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "gpqa_pass@k:k=1&n=1&strip_strings=True": 1.0 - }, - "tolerance": 0.01, - "description": "Edge case with single choice" - }, - { - "name": "Gpqa Instruct Pass At K - Multiple Correct Answers", - "metric_class": "gpqa_instruct_pass_at_k", - "metric_params": { - "k": 2, - "n": 4, - "strip_strings": true - }, - "doc": { - "query": "Which of the following are primary colors?\nA. Red\nB. Blue\nC. Green\nD. Yellow", - "choices": ["A", "B", "C", "D"], - "gold_index": 0, - "task_name": "art" - }, - "model_response": { - "text": ["A", "B", "A", "C"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "gpqa_pass@k:k=2&n=4&strip_strings=True": 0.8333333333333333 - }, - "tolerance": 0.01, - "description": "Test case with multiple correct answers (first correct answer)" - } - ] -} diff --git a/tests/unit/metrics/test_cases/loglikelihood_acc.json b/tests/unit/metrics/test_cases/loglikelihood_acc.json deleted file mode 100644 index c877566e0..000000000 --- a/tests/unit/metrics/test_cases/loglikelihood_acc.json +++ /dev/null @@ -1,266 +0,0 @@ -{ - "name": "Loglikelihood Accuracy Test Suite", - "description": "Comprehensive test cases for loglikelihood accuracy metric covering various scenarios including different logprob distributions, correct/incorrect predictions, and edge cases", - "test_cases": [ - { - "name": "Loglikelihood Accuracy - Correct Choice", - "metric_class": "loglikelihood_acc", - "metric_params": {}, - "doc": { - "query": "What is the capital of France?", - "choices": ["London", "Paris", "Berlin"], - "gold_index": 1, - "task_name": "geography" - }, - "model_response": { - "logprobs": [0.1, 0.8, 0.1], - "output_tokens": [] - }, - "expected_output": { - "acc": 1 - }, - "tolerance": 0.01, - "description": "Test loglikelihood accuracy with correct choice having highest logprob" - }, - { - "name": "Loglikelihood Accuracy - Incorrect Choice", - "metric_class": "loglikelihood_acc", - "metric_params": {}, - "doc": { - "query": "What is the largest planet in our solar system?", - "choices": ["Earth", "Jupiter", "Saturn"], - "gold_index": 1, - "task_name": "astronomy" - }, - "model_response": { - "logprobs": [0.1, 0.3, 0.6], - "output_tokens": [] - }, - "expected_output": { - "acc": 0 - }, - "tolerance": 0.01, - "description": "Test loglikelihood accuracy with incorrect choice having highest logprob" - }, - { - "name": "Loglikelihood Accuracy - Close Probabilities", - "metric_class": "loglikelihood_acc", - "metric_params": {}, - "doc": { - "query": "Who wrote Romeo and Juliet?", - "choices": ["Charles Dickens", "William Shakespeare", "Jane Austen"], - "gold_index": 1, - "task_name": "literature" - }, - "model_response": { - "logprobs": [0.2, 0.35, 0.45], - "output_tokens": [] - }, - "expected_output": { - "acc": 0 - }, - "tolerance": 0.01, - "description": "Test loglikelihood accuracy with close probabilities but wrong choice highest" - }, - { - "name": "Loglikelihood Accuracy - Very Confident Correct", - "metric_class": "loglikelihood_acc", - "metric_params": {}, - "doc": { - "query": "What is the chemical symbol for gold?", - "choices": ["Ag", "Au", "Fe"], - "gold_index": 1, - "task_name": "chemistry" - }, - "model_response": { - "logprobs": [0.01, 0.98, 0.01], - "output_tokens": [] - }, - "expected_output": { - "acc": 1 - }, - "tolerance": 0.01, - "description": "Test loglikelihood accuracy with very confident correct prediction" - }, - { - "name": "Loglikelihood Accuracy - Very Confident Incorrect", - "metric_class": "loglikelihood_acc", - "metric_params": {}, - "doc": { - "query": "What year did World War II end?", - "choices": ["1943", "1944", "1945"], - "gold_index": 2, - "task_name": "history" - }, - "model_response": { - "logprobs": [0.95, 0.03, 0.02], - "output_tokens": [] - }, - "expected_output": { - "acc": 0 - }, - "tolerance": 0.01, - "description": "Test loglikelihood accuracy with very confident incorrect prediction" - }, - { - "name": "Loglikelihood Accuracy - Equal Probabilities", - "metric_class": "loglikelihood_acc", - "metric_params": {}, - "doc": { - "query": "What is the speed of light?", - "choices": ["299,792,458 m/s", "300,000 km/s", "186,282 miles/s"], - "gold_index": 0, - "task_name": "physics" - }, - "model_response": { - "logprobs": [0.33, 0.33, 0.34], - "output_tokens": [] - }, - "expected_output": { - "acc": 0 - }, - "tolerance": 0.01, - "description": "Test loglikelihood accuracy with nearly equal probabilities" - }, - { - "name": "Loglikelihood Accuracy - Negative Logprobs", - "metric_class": "loglikelihood_acc", - "metric_params": {}, - "doc": { - "query": "How many sides does a hexagon have?", - "choices": ["4", "5", "6"], - "gold_index": 2, - "task_name": "geometry" - }, - "model_response": { - "logprobs": [-2.0, -1.5, -0.5], - "output_tokens": [] - }, - "expected_output": { - "acc": 1 - }, - "tolerance": 0.01, - "description": "Test loglikelihood accuracy with negative logprobs (correct choice highest)" - }, - { - "name": "Loglikelihood Accuracy - All Negative Logprobs", - "metric_class": "loglikelihood_acc", - "metric_params": {}, - "doc": { - "query": "What is the main theme of 1984?", - "choices": ["Love", "Totalitarianism", "War"], - "gold_index": 1, - "task_name": "literature" - }, - "model_response": { - "logprobs": [-5.0, -2.0, -4.0], - "output_tokens": [] - }, - "expected_output": { - "acc": 1 - }, - "tolerance": 0.01, - "description": "Test loglikelihood accuracy with all negative logprobs (correct choice highest)" - }, - { - "name": "Loglikelihood Accuracy - Single Choice", - "metric_class": "loglikelihood_acc", - "metric_params": {}, - "doc": { - "query": "Is the Earth round?", - "choices": ["Yes"], - "gold_index": 0, - "task_name": "science" - }, - "model_response": { - "logprobs": [0.9], - "output_tokens": [] - }, - "expected_output": { - "acc": 1 - }, - "tolerance": 0.01, - "description": "Test loglikelihood accuracy with single choice (trivial case)" - }, - { - "name": "Loglikelihood Accuracy - Multiple Gold Indices", - "metric_class": "loglikelihood_acc", - "metric_params": {}, - "doc": { - "query": "Which are primary colors?", - "choices": ["Red", "Blue", "Green", "Yellow"], - "gold_index": [0, 1], - "task_name": "art" - }, - "model_response": { - "logprobs": [0.4, 0.3, 0.2, 0.1], - "output_tokens": [] - }, - "expected_output": { - "acc": 1 - }, - "tolerance": 0.01, - "description": "Test loglikelihood accuracy with multiple correct answers (first correct answer highest)" - }, - { - "name": "Loglikelihood Accuracy - Multiple Gold Indices Wrong", - "metric_class": "loglikelihood_acc", - "metric_params": {}, - "doc": { - "query": "Which are even numbers?", - "choices": ["2", "3", "4", "5"], - "gold_index": [0, 2], - "task_name": "math" - }, - "model_response": { - "logprobs": [0.2, 0.5, 0.2, 0.1], - "output_tokens": [] - }, - "expected_output": { - "acc": 0 - }, - "tolerance": 0.01, - "description": "Test loglikelihood accuracy with multiple correct answers but wrong choice highest" - }, - { - "name": "Loglikelihood Accuracy - Zero Probabilities", - "metric_class": "loglikelihood_acc", - "metric_params": {}, - "doc": { - "query": "What is the capital of Japan?", - "choices": ["Tokyo", "Kyoto", "Osaka"], - "gold_index": 0, - "task_name": "geography" - }, - "model_response": { - "logprobs": [0.0, 0.0, 0.0], - "output_tokens": [] - }, - "expected_output": { - "acc": 1 - }, - "tolerance": 0.01, - "description": "Test loglikelihood accuracy with zero probabilities (first choice wins by default)" - }, - { - "name": "Loglikelihood Accuracy - Very Small Differences", - "metric_class": "loglikelihood_acc", - "metric_params": {}, - "doc": { - "query": "What is the largest ocean?", - "choices": ["Atlantic", "Pacific", "Indian"], - "gold_index": 1, - "task_name": "geography" - }, - "model_response": { - "logprobs": [0.333, 0.334, 0.333], - "output_tokens": [] - }, - "expected_output": { - "acc": 1 - }, - "tolerance": 0.01, - "description": "Test loglikelihood accuracy with very small differences in probabilities" - } - ] -} diff --git a/tests/unit/metrics/test_cases/loglikelihood_f1.json b/tests/unit/metrics/test_cases/loglikelihood_f1.json deleted file mode 100644 index 81a0f26cd..000000000 --- a/tests/unit/metrics/test_cases/loglikelihood_f1.json +++ /dev/null @@ -1,286 +0,0 @@ -{ - "name": "Loglikelihood F1 Test Suite", - "description": "Comprehensive test cases for loglikelihood_f1 metric covering various scenarios including different logprob distributions, correct/incorrect predictions, and edge cases. This is a corpus-level F1 score metric.", - "test_cases": [ - { - "name": "Loglikelihood F1 - Perfect Predictions", - "metric_class": "loglikelihood_f1", - "metric_params": {}, - "doc": { - "query": "What is the capital of France?", - "choices": ["London", "Paris", "Berlin"], - "gold_index": 1, - "task_name": "geography" - }, - "model_response": { - "logprobs": [0.1, 0.8, 0.1], - "output_tokens": [] - }, - "expected_output": { - "loglikelihood_f1": 1.0 - }, - "tolerance": 0.01, - "description": "Test loglikelihood F1 with perfect predictions across corpus" - }, - { - "name": "Loglikelihood F1 - All Incorrect Predictions", - "metric_class": "loglikelihood_f1", - "metric_params": {}, - "doc": { - "query": "What is the largest planet in our solar system?", - "choices": ["Earth", "Jupiter", "Saturn"], - "gold_index": 1, - "task_name": "astronomy" - }, - "model_response": { - "logprobs": [0.1, 0.3, 0.6], - "output_tokens": [] - }, - "expected_output": { - "loglikelihood_f1": 0.0 - }, - "tolerance": 0.01, - "description": "Test loglikelihood F1 with all incorrect predictions across corpus" - }, - { - "name": "Loglikelihood F1 - Mixed Predictions", - "metric_class": "loglikelihood_f1", - "metric_params": {}, - "doc": { - "query": "Who wrote Romeo and Juliet?", - "choices": ["Charles Dickens", "William Shakespeare", "Jane Austen"], - "gold_index": 1, - "task_name": "literature" - }, - "model_response": { - "logprobs": [0.2, 0.35, 0.45], - "output_tokens": [] - }, - "expected_output": { - "loglikelihood_f1": 0.0 - }, - "tolerance": 0.01, - "description": "Test loglikelihood F1 with mixed predictions (some correct, some incorrect)" - }, - { - "name": "Loglikelihood F1 - Very Confident Correct", - "metric_class": "loglikelihood_f1", - "metric_params": {}, - "doc": { - "query": "What is the chemical symbol for gold?", - "choices": ["Ag", "Au", "Fe"], - "gold_index": 1, - "task_name": "chemistry" - }, - "model_response": { - "logprobs": [0.01, 0.98, 0.01], - "output_tokens": [] - }, - "expected_output": { - "loglikelihood_f1": 1.0 - }, - "tolerance": 0.01, - "description": "Test loglikelihood F1 with very confident correct prediction" - }, - { - "name": "Loglikelihood F1 - Very Confident Incorrect", - "metric_class": "loglikelihood_f1", - "metric_params": {}, - "doc": { - "query": "What year did World War II end?", - "choices": ["1943", "1944", "1945"], - "gold_index": 2, - "task_name": "history" - }, - "model_response": { - "logprobs": [0.95, 0.03, 0.02], - "output_tokens": [] - }, - "expected_output": { - "loglikelihood_f1": 0.0 - }, - "tolerance": 0.01, - "description": "Test loglikelihood F1 with very confident incorrect prediction" - }, - { - "name": "Loglikelihood F1 - Close Probabilities", - "metric_class": "loglikelihood_f1", - "metric_params": {}, - "doc": { - "query": "What is the speed of light?", - "choices": ["299,792,458 m/s", "300,000 km/s", "186,282 miles/s"], - "gold_index": 0, - "task_name": "physics" - }, - "model_response": { - "logprobs": [0.33, 0.33, 0.34], - "output_tokens": [] - }, - "expected_output": { - "loglikelihood_f1": 0.0 - }, - "tolerance": 0.01, - "description": "Test loglikelihood F1 with close probabilities but wrong choice highest" - }, - { - "name": "Loglikelihood F1 - Negative Logprobs", - "metric_class": "loglikelihood_f1", - "metric_params": {}, - "doc": { - "query": "How many sides does a hexagon have?", - "choices": ["4", "5", "6"], - "gold_index": 2, - "task_name": "geometry" - }, - "model_response": { - "logprobs": [-2.0, -1.5, -0.5], - "output_tokens": [] - }, - "expected_output": { - "loglikelihood_f1": 1.0 - }, - "tolerance": 0.01, - "description": "Test loglikelihood F1 with negative logprobs (correct choice highest)" - }, - { - "name": "Loglikelihood F1 - All Negative Logprobs", - "metric_class": "loglikelihood_f1", - "metric_params": {}, - "doc": { - "query": "What is the main theme of 1984?", - "choices": ["Love", "Totalitarianism", "War"], - "gold_index": 1, - "task_name": "literature" - }, - "model_response": { - "logprobs": [-5.0, -2.0, -4.0], - "output_tokens": [] - }, - "expected_output": { - "loglikelihood_f1": 1.0 - }, - "tolerance": 0.01, - "description": "Test loglikelihood F1 with all negative logprobs (correct choice highest)" - }, - { - "name": "Loglikelihood F1 - Single Choice", - "metric_class": "loglikelihood_f1", - "metric_params": {}, - "doc": { - "query": "Is the Earth round?", - "choices": ["Yes"], - "gold_index": 0, - "task_name": "science" - }, - "model_response": { - "logprobs": [0.9], - "output_tokens": [] - }, - "expected_output": { - "loglikelihood_f1": 1.0 - }, - "tolerance": 0.01, - "description": "Test loglikelihood F1 with single choice (trivial case)" - }, - { - "name": "Loglikelihood F1 - Multiple Gold Indices", - "metric_class": "loglikelihood_f1", - "metric_params": {}, - "doc": { - "query": "Which are primary colors?", - "choices": ["Red", "Blue", "Green", "Yellow"], - "gold_index": [0, 1], - "task_name": "art" - }, - "model_response": { - "logprobs": [0.4, 0.3, 0.2, 0.1], - "output_tokens": [] - }, - "expected_output": { - "loglikelihood_f1": 1.0 - }, - "tolerance": 0.01, - "description": "Test loglikelihood F1 with multiple correct answers (first correct answer highest)" - }, - { - "name": "Loglikelihood F1 - Multiple Gold Indices Wrong", - "metric_class": "loglikelihood_f1", - "metric_params": {}, - "doc": { - "query": "Which are even numbers?", - "choices": ["2", "3", "4", "5"], - "gold_index": [0, 2], - "task_name": "math" - }, - "model_response": { - "logprobs": [0.2, 0.5, 0.2, 0.1], - "output_tokens": [] - }, - "expected_output": { - "loglikelihood_f1": 0.0 - }, - "tolerance": 0.01, - "description": "Test loglikelihood F1 with multiple correct answers but wrong choice highest" - }, - { - "name": "Loglikelihood F1 - Zero Probabilities", - "metric_class": "loglikelihood_f1", - "metric_params": {}, - "doc": { - "query": "What is the capital of Japan?", - "choices": ["Tokyo", "Kyoto", "Osaka"], - "gold_index": 0, - "task_name": "geography" - }, - "model_response": { - "logprobs": [0.0, 0.0, 0.0], - "output_tokens": [] - }, - "expected_output": { - "loglikelihood_f1": 1.0 - }, - "tolerance": 0.01, - "description": "Test loglikelihood F1 with zero probabilities (first choice wins by default)" - }, - { - "name": "Loglikelihood F1 - Very Small Differences", - "metric_class": "loglikelihood_f1", - "metric_params": {}, - "doc": { - "query": "What is the largest ocean?", - "choices": ["Atlantic", "Pacific", "Indian"], - "gold_index": 1, - "task_name": "geography" - }, - "model_response": { - "logprobs": [0.333, 0.334, 0.333], - "output_tokens": [] - }, - "expected_output": { - "loglikelihood_f1": 1.0 - }, - "tolerance": 0.01, - "description": "Test loglikelihood F1 with very small differences in probabilities" - }, - { - "name": "Loglikelihood F1 - Balanced Predictions", - "metric_class": "loglikelihood_f1", - "metric_params": {}, - "doc": { - "query": "What is the square root of 16?", - "choices": ["2", "4", "8"], - "gold_index": 1, - "task_name": "math" - }, - "model_response": { - "logprobs": [0.25, 0.5, 0.25], - "output_tokens": [] - }, - "expected_output": { - "loglikelihood_f1": 1.0 - }, - "tolerance": 0.01, - "description": "Test loglikelihood F1 with balanced predictions (correct choice has highest probability)" - } - ] -} diff --git a/tests/unit/metrics/test_cases/maj_at_k.json b/tests/unit/metrics/test_cases/maj_at_k.json deleted file mode 100644 index 031735938..000000000 --- a/tests/unit/metrics/test_cases/maj_at_k.json +++ /dev/null @@ -1,82 +0,0 @@ -{ - "name": "Maj At K Test Suite", - "description": "Test cases for maj_at_k metric", - "test_cases": [ - { - "name": "Maj at K - Majority Correct", - "metric_class": "maj_at_k", - "metric_params": {"k": 3}, - "doc": { - "query": "What is the capital of France?", - "choices": ["London", "Paris", "Berlin"], - "gold_index": [1], - "task_name": "geography" - }, - "model_response": { - "text": ["Paris", "Paris", "London"] - }, - "expected_output": { - "maj@k:k=3": 1 - }, - "tolerance": 0.01, - "description": "Test maj at k with majority correct" - }, - { - "name": "Maj at K - No Majority", - "metric_class": "maj_at_k", - "metric_params": {"k": 3}, - "doc": { - "query": "What is the capital of France?", - "choices": ["London", "Paris", "Berlin"], - "gold_index": [1], - "task_name": "geography" - }, - "model_response": { - "text": ["Paris", "London", "Berlin"] - }, - "expected_output": { - "maj@k:k=3": 1 - }, - "tolerance": 0.01, - "description": "Test maj at k with no majority" - }, - { - "name": "Maj at K - All Correct", - "metric_class": "maj_at_k", - "metric_params": {"k": 3}, - "doc": { - "query": "What is the capital of France?", - "choices": ["London", "Paris", "Berlin"], - "gold_index": [1], - "task_name": "geography" - }, - "model_response": { - "text": ["Paris", "Paris", "Paris"] - }, - "expected_output": { - "maj@k:k=3": 1 - }, - "tolerance": 0.01, - "description": "Test maj at k with all correct" - }, - { - "name": "Maj at K - Wrong Answer", - "metric_class": "maj_at_k", - "metric_params": {"k": 3}, - "doc": { - "query": "What is the capital of France?", - "choices": ["London", "Paris", "Berlin"], - "gold_index": [1], - "task_name": "geography" - }, - "model_response": { - "text": ["London", "London", "London"] - }, - "expected_output": { - "maj@k:k=3": 0 - }, - "tolerance": 0.01, - "description": "Test maj at k with wrong answer" - } - ] -} diff --git a/tests/unit/metrics/test_cases/mcc.json b/tests/unit/metrics/test_cases/mcc.json deleted file mode 100644 index b0cbaa219..000000000 --- a/tests/unit/metrics/test_cases/mcc.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "name": "MCC Test Suite", - "description": "Test cases for MCC (Matthews Correlation Coefficient) metric", - "corpus_level": true, - "test_cases": [ - { - "name": "MCC - Corpus Level Test with 3 Samples", - "metric_class": "mcc", - "metric_name": "mcc", - "metric_params": {}, - "docs": [ - { - "query": "What is the capital of France?", - "choices": ["Paris", "London", "Berlin"], - "gold_index": 0, - "task_name": "geography" - }, - { - "query": "What is 2 + 2?", - "choices": ["3", "4", "5"], - "gold_index": 1, - "task_name": "math" - }, - { - "query": "What color is the sky?", - "choices": ["Red", "Blue", "Green"], - "gold_index": 1, - "task_name": "science" - } - ], - "model_responses": [ - { - "logprobs": [-0.2, -0.8, -1.5] - }, - { - "logprobs": [-1.2, -0.3, -0.9] - }, - { - "logprobs": [-0.7, -0.4, -1.1] - } - ], - "expected_output": 1.0, - "tolerance": 0.01, - "description": "Corpus level test case for MCC metric with 3 samples - all predictions correct" - } - ] -} diff --git a/tests/unit/metrics/test_cases/mrr.json b/tests/unit/metrics/test_cases/mrr.json deleted file mode 100644 index 0fe43dca4..000000000 --- a/tests/unit/metrics/test_cases/mrr.json +++ /dev/null @@ -1,90 +0,0 @@ -{ - "name": "Mrr Test Suite", - "description": "Test cases for mrr metric", - "test_cases": [ - { - "name": "MRR - Correct First", - "metric_class": "mrr", - "metric_params": {}, - "doc": { - "query": "What is the capital of France?", - "choices": ["London", "Paris", "Berlin"], - "gold_index": 1, - "task_name": "geography" - }, - "model_response": { - "text": ["Paris"], - "logprobs": [0.1, 0.8, 0.1], - "output_tokens": [] - }, - "expected_output": { - "mrr": 1.0 - }, - "tolerance": 0.01, - "description": "Test MRR with correct choice ranked first" - }, - { - "name": "MRR - Correct Second", - "metric_class": "mrr", - "metric_params": {}, - "doc": { - "query": "What is the capital of France?", - "choices": ["London", "Paris", "Berlin"], - "gold_index": 1, - "task_name": "geography" - }, - "model_response": { - "text": ["London"], - "logprobs": [0.8, 0.1, 0.1], - "output_tokens": [] - }, - "expected_output": { - "mrr": 0.5 - }, - "tolerance": 0.01, - "description": "Test MRR with correct choice ranked second" - }, - { - "name": "MRR - Correct Third", - "metric_class": "mrr", - "metric_params": {}, - "doc": { - "query": "What is the capital of France?", - "choices": ["London", "Berlin", "Paris"], - "gold_index": 2, - "task_name": "geography" - }, - "model_response": { - "text": ["London"], - "logprobs": [0.8, 0.15, 0.05], - "output_tokens": [] - }, - "expected_output": { - "mrr": 0.3333333333333333 - }, - "tolerance": 0.01, - "description": "Test MRR with correct choice ranked third" - }, - { - "name": "MRR - Multiple Gold Indices", - "metric_class": "mrr", - "metric_params": {}, - "doc": { - "query": "Which are European capitals?", - "choices": ["London", "Paris", "Tokyo", "Berlin"], - "gold_index": [0, 1, 3], - "task_name": "geography" - }, - "model_response": { - "text": ["Paris"], - "logprobs": [0.2, 0.6, 0.1, 0.1], - "output_tokens": [] - }, - "expected_output": { - "mrr": 1.0 - }, - "tolerance": 0.01, - "description": "Test MRR with multiple gold indices" - } - ] -} diff --git a/tests/unit/metrics/test_cases/multi_f1_numeric.json b/tests/unit/metrics/test_cases/multi_f1_numeric.json deleted file mode 100644 index ccc0ac536..000000000 --- a/tests/unit/metrics/test_cases/multi_f1_numeric.json +++ /dev/null @@ -1,167 +0,0 @@ -{ - "name": "Multi F1 Numeric Test Suite", - "description": "Test cases for multi_f1_numeric metric (corpus-level multi-class F1 score with 3 classes)", - "corpus_level": true, - "test_cases": [ - { - "name": "Multi F1 Numeric - Perfect Predictions", - "metric_class": "multi_f1_numeric", - "metric_params": {}, - "metric_name": "mf1", - "docs": [ - { - "query": "Classify the sentiment: I love this movie!", - "choices": ["negative", "neutral", "positive"], - "gold_index": 2, - "task_name": "sentiment_classification" - }, - { - "query": "Classify the topic: 2 + 2 = 4", - "choices": ["history", "science", "math"], - "gold_index": 2, - "task_name": "topic_classification" - }, - { - "query": "Classify the emotion: I am so happy today!", - "choices": ["sad", "angry", "happy"], - "gold_index": 2, - "task_name": "emotion_classification" - } - ], - "model_responses": [ - { - "logprobs": [-2.0, -1.5, -0.1] - }, - { - "logprobs": [-1.8, -2.1, -0.2] - }, - { - "logprobs": [-2.2, -1.9, -0.1] - } - ], - "expected_output": 1.0, - "tolerance": 0.01, - "description": "Perfect predictions - all classes correctly predicted (F1 = 1.0 for each class)" - }, - { - "name": "Multi F1 Numeric - Balanced Performance", - "metric_class": "multi_f1_numeric", - "metric_params": {}, - "metric_name": "mf1", - "docs": [ - { - "query": "Classify the sentiment: The weather is okay", - "choices": ["negative", "neutral", "positive"], - "gold_index": 1, - "task_name": "sentiment_classification" - }, - { - "query": "Classify the topic: The French Revolution", - "choices": ["history", "science", "math"], - "gold_index": 0, - "task_name": "topic_classification" - }, - { - "query": "Classify the emotion: I feel nothing special", - "choices": ["sad", "angry", "happy"], - "gold_index": 0, - "task_name": "emotion_classification" - } - ], - "model_responses": [ - { - "logprobs": [-1.0, -0.2, -1.5] - }, - { - "logprobs": [-0.1, -1.8, -2.0] - }, - { - "logprobs": [-0.2, -1.5, -1.8] - } - ], - "expected_output": 1.0, - "tolerance": 0.01, - "description": "Balanced performance - 2 correct, 1 incorrect (F1 varies by class)" - }, - { - "name": "Multi F1 Numeric - Poor Performance", - "metric_class": "multi_f1_numeric", - "metric_params": {}, - "metric_name": "mf1", - "docs": [ - { - "query": "Classify the sentiment: This is terrible", - "choices": ["negative", "neutral", "positive"], - "gold_index": 0, - "task_name": "sentiment_classification" - }, - { - "query": "Classify the topic: Photosynthesis", - "choices": ["history", "science", "math"], - "gold_index": 1, - "task_name": "topic_classification" - }, - { - "query": "Classify the emotion: I am furious", - "choices": ["sad", "angry", "happy"], - "gold_index": 1, - "task_name": "emotion_classification" - } - ], - "model_responses": [ - { - "logprobs": [-1.5, -0.1, -0.8] - }, - { - "logprobs": [-0.2, -1.8, -0.3] - }, - { - "logprobs": [-0.1, -1.9, -0.2] - } - ], - "expected_output": 0.33, - "tolerance": 0.01, - "description": "Poor performance - 1 correct, 2 incorrect (low F1 across classes)" - }, - { - "name": "Multi F1 Numeric - Random Performance", - "metric_class": "multi_f1_numeric", - "metric_params": {}, - "metric_name": "mf1", - "docs": [ - { - "query": "Classify the sentiment: I don't know", - "choices": ["negative", "neutral", "positive"], - "gold_index": 1, - "task_name": "sentiment_classification" - }, - { - "query": "Classify the topic: Calculus", - "choices": ["history", "science", "math"], - "gold_index": 2, - "task_name": "topic_classification" - }, - { - "query": "Classify the emotion: I am confused", - "choices": ["sad", "angry", "happy"], - "gold_index": 0, - "task_name": "emotion_classification" - } - ], - "model_responses": [ - { - "logprobs": [-0.5, -0.5, -0.5] - }, - { - "logprobs": [-0.5, -0.5, -0.5] - }, - { - "logprobs": [-0.5, -0.5, -0.5] - } - ], - "expected_output": 0.55, - "tolerance": 0.1, - "description": "Random performance - equal logprobs lead to random predictions (F1 ≈ 0.0)" - } - ] -} diff --git a/tests/unit/metrics/test_cases/pass_at_k.json b/tests/unit/metrics/test_cases/pass_at_k.json deleted file mode 100644 index 959b194fa..000000000 --- a/tests/unit/metrics/test_cases/pass_at_k.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "name": "Pass At K Test Suite", - "description": "Test cases for pass_at_k metric", - "test_cases": [ - { - "name": "Pass at K - Correct in K", - "metric_class": "pass_at_k", - "metric_params": {"k": 1, "n": 2}, - "doc": { - "query": "What is the capital of France?", - "choices": ["London", "Paris", "Berlin"], - "gold_index": 1, - "task_name": "geography" - }, - "model_response": { - "text": ["Paris", "London"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "pass@k:k=1&n=2": 0.5 - }, - "tolerance": 0.01, - "description": "Test pass at k with correct answer in k" - }, - { - "name": "Pass at K - Not in K", - "metric_class": "pass_at_k", - "metric_params": {"k": 1, "n": 2}, - "doc": { - "query": "What is the capital of France?", - "choices": ["London", "Paris", "Berlin"], - "gold_index": 1, - "task_name": "geography" - }, - "model_response": { - "text": ["London", "Berlin"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "pass@k:k=1&n=2": 0.0 - }, - "tolerance": 0.01, - "description": "Test pass at k with correct answer not in k" - }, - { - "name": "Pass at K - Multiple Attempts", - "metric_class": "pass_at_k", - "metric_params": {"k": 2, "n": 3}, - "doc": { - "query": "What is the capital of France?", - "choices": ["London", "Paris", "Berlin"], - "gold_index": 1, - "task_name": "geography" - }, - "model_response": { - "text": ["London", "Paris", "Berlin"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "pass@k:k=2&n=3": 0.66 - }, - "tolerance": 0.01, - "description": "Test pass at k with multiple attempts" - } - ] -} diff --git a/tests/unit/metrics/test_cases/pass_at_k_letters.json b/tests/unit/metrics/test_cases/pass_at_k_letters.json deleted file mode 100644 index 6b64f738d..000000000 --- a/tests/unit/metrics/test_cases/pass_at_k_letters.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "name": "Pass At K Letters Test Suite", - "description": "Test cases for pass_at_k_letters metric", - "test_cases": [ - { - "name": "Pass at K Letters - Correct Letters", - "metric_class": "pass_at_k_letters", - "metric_params": {"k": 1, "n": 2}, - "doc": { - "query": "What letter comes after A?", - "choices": ["B"], - "gold_index": 0, - "task_name": "alphabet" - }, - "model_response": { - "text": ["B", "C"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "pass@k:k=1&n=2": 0.0 - }, - "tolerance": 0.01, - "description": "Test pass at k letters with correct letter answer" - }, - { - "name": "Pass at K Letters - Wrong Letters", - "metric_class": "pass_at_k_letters", - "metric_params": {"k": 1, "n": 2}, - "doc": { - "query": "What letter comes after A?", - "choices": ["B"], - "gold_index": 0, - "task_name": "alphabet" - }, - "model_response": { - "text": ["C", "D"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "pass@k:k=1&n=2": 0.0 - }, - "tolerance": 0.01, - "description": "Test pass at k letters with wrong letter answer" - }, - { - "name": "Pass at K Letters - Multiple Attempts", - "metric_class": "pass_at_k_letters", - "metric_params": {"k": 2, "n": 3}, - "doc": { - "query": "What letter comes after B?", - "choices": ["C"], - "gold_index": 0, - "task_name": "alphabet" - }, - "model_response": { - "text": ["D", "C", "E"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "pass@k:k=2&n=3": 0.0 - }, - "tolerance": 0.01, - "description": "Test pass at k letters with multiple attempts" - } - ] -} diff --git a/tests/unit/metrics/test_cases/pass_at_k_math.json b/tests/unit/metrics/test_cases/pass_at_k_math.json deleted file mode 100644 index b6aac1749..000000000 --- a/tests/unit/metrics/test_cases/pass_at_k_math.json +++ /dev/null @@ -1,63 +0,0 @@ -{ - "name": "Pass At K Math Test Suite", - "description": "Test cases for pass_at_k_math metric", - "test_cases": [ - { - "name": "Pass at K Math - Correct Math", - "metric_class": "pass_at_k_math", - "metric_params": {"k": 1, "n": 2}, - "doc": { - "query": "What is 2 + 2?", - "choices": ["4"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["4", "5"] - }, - "expected_output": { - "pass@k:k=1&n=2": 0.5 - }, - "tolerance": 0.01, - "description": "Test pass at k math with correct math answer" - }, - { - "name": "Pass at K Math - Wrong Math", - "metric_class": "pass_at_k_math", - "metric_params": {"k": 1, "n": 2}, - "doc": { - "query": "What is 2 + 2?", - "choices": ["4"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["5", "6"] - }, - "expected_output": { - "pass@k:k=1&n=2": 0.0 - }, - "tolerance": 0.01, - "description": "Test pass at k math with wrong math answer" - }, - { - "name": "Pass at K Math - Multiple Attempts", - "metric_class": "pass_at_k_math", - "metric_params": {"k": 2, "n": 3}, - "doc": { - "query": "What is 3 * 4?", - "choices": ["12"], - "gold_index": 0, - "task_name": "math" - }, - "model_response": { - "text": ["10", "12", "15"] - }, - "expected_output": { - "pass@k:k=2&n=3": 0.66 - }, - "tolerance": 0.01, - "description": "Test pass at k math with multiple attempts" - } - ] -} diff --git a/tests/unit/metrics/test_cases/prediction_perplexity.json b/tests/unit/metrics/test_cases/prediction_perplexity.json deleted file mode 100644 index 26468edcc..000000000 --- a/tests/unit/metrics/test_cases/prediction_perplexity.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "name": "Prediction Perplexity Test Suite", - "description": "Test cases for prediction_perplexity metric", - "test_cases": [ - { - "name": "Prediction Perplexity - Basic Test", - "metric_class": "prediction_perplexity", - "metric_params": {}, - "doc": { - "query": "Test query for prediction_perplexity", - "choices": [ - "Test choice 1", - "Test choice 2", - "Test choice 3" - ], - "gold_index": 0, - "task_name": "test" - }, - "model_response": { - "text": [ - "Test choice 1" - ], - "logprobs": [ - 0.5, - 0.3, - 0.2 - ], - "output_tokens": [ - [ - 1 - ], - [ - 2 - ], - [ - 3 - ] - ] - }, - "expected_output": { - "ppl": 1.0 - }, - "tolerance": 0.01, - "description": "Basic test case for prediction_perplexity metric" - } - ] -} diff --git a/tests/unit/metrics/test_cases/recall_at_k.json b/tests/unit/metrics/test_cases/recall_at_k.json deleted file mode 100644 index 149466bf9..000000000 --- a/tests/unit/metrics/test_cases/recall_at_k.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "name": "Recall At K Test Suite", - "description": "Test cases for recall_at_k metric", - "test_cases": [ - { - "name": "Recall At K - Correct in Top K", - "metric_class": "recall_at_k", - "metric_params": {"k": 2}, - "doc": { - "query": "What is the capital of France?", - "choices": ["London", "Paris", "Berlin", "Madrid"], - "gold_index": 1, - "task_name": "geography" - }, - "model_response": { - "text": ["Paris"], - "logprobs": [0.1, 0.8, 0.05, 0.05], - "output_tokens": [] - }, - "expected_output": { - "recall:k=2": 1 - }, - "tolerance": 0.01, - "description": "Test recall at k with correct choice in top k" - }, - { - "name": "Recall At K - Not in Top K", - "metric_class": "recall_at_k", - "metric_params": {"k": 1}, - "doc": { - "query": "What is the capital of France?", - "choices": ["London", "Paris", "Berlin", "Madrid"], - "gold_index": 1, - "task_name": "geography" - }, - "model_response": { - "text": ["London"], - "logprobs": [0.8, 0.1, 0.05, 0.05], - "output_tokens": [] - }, - "expected_output": { - "recall:k=1": 0 - }, - "tolerance": 0.01, - "description": "Test recall at k with correct choice not in top k" - }, - { - "name": "Recall At K - Multiple Gold Indices", - "metric_class": "recall_at_k", - "metric_params": {"k": 2}, - "doc": { - "query": "Which are European capitals?", - "choices": ["London", "Paris", "Tokyo", "Berlin"], - "gold_index": [0, 1, 3], - "task_name": "geography" - }, - "model_response": { - "text": ["Paris", "London"], - "logprobs": [0.3, 0.4, 0.1, 0.2], - "output_tokens": [] - }, - "expected_output": { - "recall:k=2": 1 - }, - "tolerance": 0.01, - "description": "Test recall at k with multiple gold indices" - } - ] -} diff --git a/tests/unit/metrics/test_cases/rouge1.json b/tests/unit/metrics/test_cases/rouge1.json deleted file mode 100644 index f937a4de5..000000000 --- a/tests/unit/metrics/test_cases/rouge1.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "ROUGE1 Test Suite", - "description": "Test cases for ROUGE1 metric", - "test_cases": [ - { - "name": "ROUGE Score", - "metric_class": "rouge1", - "metric_params": { - }, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "test" - }, - "model_response": { - "text": ["The quick brown fox jumps over the lazy dog"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "rouge1": 1 - }, - "tolerance": 0.01, - "description": "Test ROUGE score with perfect match" - } - ] -} diff --git a/tests/unit/metrics/test_cases/rouge2.json b/tests/unit/metrics/test_cases/rouge2.json deleted file mode 100644 index f18e1ca3a..000000000 --- a/tests/unit/metrics/test_cases/rouge2.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "name": "Rouge2 Test Suite", - "description": "Test cases for rouge2 metric", - "test_cases": [ - { - "name": "ROUGE2 - Perfect Match", - "metric_class": "rouge2", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["The quick brown fox jumps over the lazy dog"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "rouge2": 1.0 - }, - "tolerance": 0.01, - "description": "Test ROUGE2 with perfect match" - }, - { - "name": "ROUGE2 - Partial Match", - "metric_class": "rouge2", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["The quick brown fox"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "rouge2": 0.5454 - }, - "tolerance": 0.01, - "description": "Test ROUGE2 with partial match (no bigram overlap)" - }, - { - "name": "ROUGE2 - Some Bigram Overlap", - "metric_class": "rouge2", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["The quick brown fox jumps"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "rouge2": 0.666 - }, - "tolerance": 0.1, - "description": "Test ROUGE2 with some bigram overlap" - } - ] -} diff --git a/tests/unit/metrics/test_cases/rougeL.json b/tests/unit/metrics/test_cases/rougeL.json deleted file mode 100644 index 81635aa05..000000000 --- a/tests/unit/metrics/test_cases/rougeL.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "name": "Rougel Test Suite", - "description": "Test cases for rougeL metric", - "test_cases": [ - { - "name": "ROUGEL - Perfect Match", - "metric_class": "rougeL", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["The quick brown fox jumps over the lazy dog"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "rougeL": 1.0 - }, - "tolerance": 0.01, - "description": "Test ROUGEL with perfect match" - }, - { - "name": "ROUGEL - Partial Match", - "metric_class": "rougeL", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["The quick brown fox"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "rougeL": 0.615 - }, - "tolerance": 0.1, - "description": "Test ROUGEL with partial match" - }, - { - "name": "ROUGEL - Different Word Order", - "metric_class": "rougeL", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["The brown quick fox jumps over the dog lazy"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "rougeL": 0.8 - }, - "tolerance": 0.1, - "description": "Test ROUGEL with different word order" - } - ] -} diff --git a/tests/unit/metrics/test_cases/rougeLsum.json b/tests/unit/metrics/test_cases/rougeLsum.json deleted file mode 100644 index 8a5faf3a3..000000000 --- a/tests/unit/metrics/test_cases/rougeLsum.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "name": "Rougelsum Test Suite", - "description": "Test cases for rougeLsum metric", - "test_cases": [ - { - "name": "ROUGELsum - Perfect Match", - "metric_class": "rougeLsum", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["The quick brown fox jumps over the lazy dog"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "rougeLsum": 1.0 - }, - "tolerance": 0.01, - "description": "Test ROUGELsum with perfect match" - }, - { - "name": "ROUGELsum - Partial Match", - "metric_class": "rougeLsum", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["The quick brown fox"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "rougeLsum": 0.61 - }, - "tolerance": 0.1, - "description": "Test ROUGELsum with partial match" - }, - { - "name": "ROUGELsum - Multi-sentence", - "metric_class": "rougeLsum", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog. The fox is very fast."], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["The quick brown fox jumps over the lazy dog. The fox is very fast."], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "rougeLsum": 1.0 - }, - "tolerance": 0.01, - "description": "Test ROUGELsum with multi-sentence text" - } - ] -} diff --git a/tests/unit/metrics/test_cases/rouge_t5.json b/tests/unit/metrics/test_cases/rouge_t5.json deleted file mode 100644 index df2f81777..000000000 --- a/tests/unit/metrics/test_cases/rouge_t5.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "name": "Rouge T5 Test Suite", - "description": "Test cases for rouge_t5 metric", - "test_cases": [ - { - "name": "ROUGE T5 - Perfect Match", - "metric_class": "rouge_t5", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["The quick brown fox jumps over the lazy dog"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "rouge1": 100.0, - "rouge2": 100.0, - "rougeL": 100.0, - "rougeLsum": 100.0 - }, - "tolerance": 0.01, - "description": "Test ROUGE T5 with perfect match" - }, - { - "name": "ROUGE T5 - Partial Match", - "metric_class": "rouge_t5", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["The quick brown fox"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "rouge1": 61.53846153846153, - "rouge2": 54.54545454545454, - "rougeL": 61.53846153846153, - "rougeLsum": 61.53846153846153 - }, - "tolerance": 0.1, - "description": "Test ROUGE T5 with partial match" - }, - { - "name": "ROUGE T5 - Different Content", - "metric_class": "rouge_t5", - "metric_params": {}, - "doc": { - "query": "Summarize the text", - "choices": ["The quick brown fox jumps over the lazy dog"], - "gold_index": 0, - "task_name": "summarization" - }, - "model_response": { - "text": ["A cat sleeps on the mat"], - "logprobs": [], - "output_tokens": [] - }, - "expected_output": { - "rouge1": 13.333333333333334, - "rouge2": 0.0, - "rougeL": 13.333333333333334, - "rougeLsum": 13.333333333333334 - }, - "tolerance": 0.01, - "description": "Test ROUGE T5 with completely different content" - } - ] -} diff --git a/tests/unit/metrics/test_cases/simpleqa_judge.json b/tests/unit/metrics/test_cases/simpleqa_judge.json deleted file mode 100644 index 485bf4b3d..000000000 --- a/tests/unit/metrics/test_cases/simpleqa_judge.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "name": "Simpleqa Judge Test Suite", - "description": "Test cases for simpleqa_judge metric", - "test_cases": [ - { - "name": "Simpleqa Judge - Basic Test", - "metric_class": "simpleqa_judge", - "metric_params": {}, - "doc": { - "query": "Test query for simpleqa_judge", - "choices": [ - "Test choice 1", - "Test choice 2", - "Test choice 3" - ], - "gold_index": 0, - "task_name": "test" - }, - "model_response": { - "text": [ - "Test choice 1" - ] - }, - "expected_output": { - "simpleqa_judge": 1.0 - }, - "tolerance": 0.01, - "description": "Basic test case for simpleqa_judge metric" - } - ] -} diff --git a/tests/unit/metrics/test_cases/target_perplexity.json b/tests/unit/metrics/test_cases/target_perplexity.json deleted file mode 100644 index 5654613c2..000000000 --- a/tests/unit/metrics/test_cases/target_perplexity.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "name": "Target Perplexity Test Suite", - "description": "Test cases for target_perplexity metric (sample-level perplexity of target text)", - "test_cases": [ - { - "name": "Target Perplexity - Low Perplexity", - "metric_class": "target_perplexity", - "metric_params": {}, - "doc": { - "query": "What is the capital of France?", - "choices": ["Paris", "London", "Berlin"], - "gold_index": 0, - "task_name": "geography" - }, - "model_response": { - "logprobs": [-0.1, -0.2, -0.3] - }, - "expected_output": { - "ppl": 1.5 - }, - "tolerance": 0.01, - "description": "Low perplexity - model has high confidence in target text" - }, - { - "name": "Target Perplexity - Moderate Perplexity", - "metric_class": "target_perplexity", - "metric_params": {}, - "doc": { - "query": "What is 2 + 2?", - "choices": ["3", "4", "5"], - "gold_index": 1, - "task_name": "math" - }, - "model_response": { - "logprobs": [-0.8, -0.3, -1.2] - }, - "expected_output": { - "ppl": 2.0 - }, - "tolerance": 0.01, - "description": "Moderate perplexity - model has moderate confidence in target text" - }, - { - "name": "Target Perplexity - High Perplexity", - "metric_class": "target_perplexity", - "metric_params": {}, - "doc": { - "query": "What color is the sky?", - "choices": ["Red", "Blue", "Green"], - "gold_index": 1, - "task_name": "science" - }, - "model_response": { - "logprobs": [-1.5, -0.1, -1.8] - }, - "expected_output": { - "ppl": 0.0 - }, - "tolerance": 0.01, - "description": "High perplexity - model has low confidence in target text" - }, - { - "name": "Target Perplexity - Very High Perplexity", - "metric_class": "target_perplexity", - "metric_params": {}, - "doc": { - "query": "What is the largest planet?", - "choices": ["Mars", "Jupiter", "Saturn"], - "gold_index": 1, - "task_name": "astronomy" - }, - "model_response": { - "logprobs": [-2.1, -0.2, -2.5] - }, - "expected_output": { - "ppl": 8.2 - }, - "tolerance": 0.8, - "description": "Very high perplexity - model has very low confidence in target text" - }, - { - "name": "Target Perplexity - Mixed Confidence", - "metric_class": "target_perplexity", - "metric_params": {}, - "doc": { - "query": "What is the weather like?", - "choices": ["Sunny", "Rainy", "Cloudy"], - "gold_index": 0, - "task_name": "weather" - }, - "model_response": { - "logprobs": [-0.2, -1.8, -1.5] - }, - "expected_output": { - "ppl": 1.2 - }, - "tolerance": 0.2, - "description": "Mixed confidence - high confidence in correct choice, low in others" - } - ] -} diff --git a/tests/unit/metrics/test_cases/ter.json b/tests/unit/metrics/test_cases/ter.json deleted file mode 100644 index 39b671b0f..000000000 --- a/tests/unit/metrics/test_cases/ter.json +++ /dev/null @@ -1,167 +0,0 @@ -{ - "name": "TER Test Suite", - "description": "Test cases for ter metric (Translation Edit Rate - corpus-level)", - "corpus_level": true, - "test_cases": [ - { - "name": "TER - Perfect Translations", - "metric_class": "ter", - "metric_params": {}, - "metric_name": "ter", - "docs": [ - { - "query": "Translate to French: Hello world", - "choices": ["Bonjour le monde"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to Spanish: Good morning", - "choices": ["Buenos días"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to German: Thank you", - "choices": ["Danke schön"], - "gold_index": 0, - "task_name": "translation" - } - ], - "model_responses": [ - { - "text": ["Bonjour le monde"] - }, - { - "text": ["Buenos días"] - }, - { - "text": ["Danke schön"] - } - ], - "expected_output": 0.0, - "tolerance": 0.01, - "description": "Perfect translations - no edits needed (TER = 0.0)" - }, - { - "name": "TER - Minor Edits", - "metric_class": "ter", - "metric_params": {}, - "metric_name": "ter", - "docs": [ - { - "query": "Translate to French: The cat is sleeping", - "choices": ["Le chat dort"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to Spanish: I like pizza", - "choices": ["Me gusta la pizza"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to German: The weather is nice", - "choices": ["Das Wetter ist schön"], - "gold_index": 0, - "task_name": "translation" - } - ], - "model_responses": [ - { - "text": ["Le chat dort"] - }, - { - "text": ["Me gusta pizza"] - }, - { - "text": ["Das Wetter ist schön"] - } - ], - "expected_output": 0.0, - "tolerance": 0.05, - "description": "Minor edits - small word differences" - }, - { - "name": "TER - Major Edits", - "metric_class": "ter", - "metric_params": {}, - "metric_name": "ter", - "docs": [ - { - "query": "Translate to French: The quick brown fox jumps over the lazy dog", - "choices": ["Le renard brun rapide saute par-dessus le chien paresseux"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to Spanish: Artificial intelligence is transforming the world", - "choices": ["La inteligencia artificial está transformando el mundo"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to German: Machine learning algorithms are becoming more sophisticated", - "choices": ["Maschinelle Lernalgorithmen werden immer ausgefeilter"], - "gold_index": 0, - "task_name": "translation" - } - ], - "model_responses": [ - { - "text": ["Le renard saute le chien"] - }, - { - "text": ["La IA cambia el mundo"] - }, - { - "text": ["ML Algorithmen werden besser"] - } - ], - "expected_output": 57.14285714285714, - "tolerance": 0.01, - "description": "Major edits - significant word omissions and changes" - }, - { - "name": "TER - Completely Different", - "metric_class": "ter", - "metric_params": {}, - "metric_name": "ter", - "docs": [ - { - "query": "Translate to French: The sun is bright today", - "choices": ["Le soleil est brillant aujourd'hui"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to Spanish: The moon is full tonight", - "choices": ["La luna está llena esta noche"], - "gold_index": 0, - "task_name": "translation" - }, - { - "query": "Translate to German: The stars are beautiful", - "choices": ["Die Sterne sind wunderschön"], - "gold_index": 0, - "task_name": "translation" - } - ], - "model_responses": [ - { - "text": ["Il pleut beaucoup"] - }, - { - "text": ["Hace mucho frío"] - }, - { - "text": ["Es ist sehr heiß"] - } - ], - "expected_output": 80.0, - "tolerance": 0.1, - "description": "Completely different translations - maximum edit distance" - } - ] -} diff --git a/tests/unit/metrics/test_cases/truthfulqa_mc_metrics.json b/tests/unit/metrics/test_cases/truthfulqa_mc_metrics.json deleted file mode 100644 index 29a009b0d..000000000 --- a/tests/unit/metrics/test_cases/truthfulqa_mc_metrics.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "name": "Truthfulqa Mc Metrics Test Suite", - "description": "Test cases for truthfulqa_mc_metrics metric", - "test_cases": [ - { - "name": "TruthfulQA MC Metrics - Correct Answer", - "metric_class": "truthfulqa_mc_metrics", - "metric_params": {}, - "doc": { - "query": "Q: What is the capital of France?\nA:", - "choices": [" Paris", " London", " Berlin", " Madrid", " Rome"], - "gold_index": [0, 3, 4], - "specific": { - "len_mc1": 3 - }, - "task_name": "truthfulqa" - }, - "model_response": { - "text": [" Paris"], - "logprobs": [0.8, 0.1, 0.1, 0.0, 0.0], - "output_tokens": [] - }, - "expected_output": { - "truthfulqa_mc1": 1.0, - "truthfulqa_mc2": 1.0 - }, - "tolerance": 0.01, - "description": "Test TruthfulQA MC metrics with correct answer" - }, - { - "name": "TruthfulQA MC Metrics - Wrong Answer", - "metric_class": "truthfulqa_mc_metrics", - "metric_params": {}, - "doc": { - "query": "Q: What is the capital of France?\nA:", - "choices": [" Paris", " London", " Berlin", " Madrid", " Rome"], - "gold_index": [0, 3, 4], - "specific": { - "len_mc1": 3 - }, - "task_name": "truthfulqa" - }, - "model_response": { - "text": [" London"], - "logprobs": [0.1, 0.8, 0.1, 0.0, 0.0], - "output_tokens": [] - }, - "expected_output": { - "truthfulqa_mc1": false, - "truthfulqa_mc2": 1.0 - }, - "tolerance": 0.01, - "description": "Test TruthfulQA MC metrics with wrong answer" - }, - { - "name": "TruthfulQA MC Metrics - Multiple Gold Indices", - "metric_class": "truthfulqa_mc_metrics", - "metric_params": {}, - "doc": { - "query": "Q: Which are European capitals?\nA:", - "choices": [" London", " Paris", " Tokyo", " Berlin", " Madrid", " Rome"], - "gold_index": [0, 1, 3, 4, 5], - "specific": { - "len_mc1": 4 - }, - "task_name": "truthfulqa" - }, - "model_response": { - "text": [" Paris"], - "logprobs": [0.1, 0.6, 0.1, 0.1, 0.05, 0.05], - "output_tokens": [] - }, - "expected_output": { - "truthfulqa_mc1": false, - "truthfulqa_mc2": 0.0 - }, - "tolerance": 0.01, - "description": "Test TruthfulQA MC metrics with multiple gold indices" - } - ] -} diff --git a/tests/unit/metrics/test_cases/word_perplexity.json b/tests/unit/metrics/test_cases/word_perplexity.json deleted file mode 100644 index 4f4640e67..000000000 --- a/tests/unit/metrics/test_cases/word_perplexity.json +++ /dev/null @@ -1,127 +0,0 @@ -{ - "name": "Word Perplexity Test Suite", - "description": "Test cases for word_perplexity metric (corpus-level weighted perplexity)", - "corpus_level": true, - "test_cases": [ - { - "name": "Word Perplexity - Low Perplexity", - "metric_class": "word_perplexity", - "metric_params": {}, - "metric_name": "word_perplexity", - "docs": [ - { - "query": "The quick brown fox", - "choices": ["jumps over the lazy dog"], - "gold_index": 0, - "task_name": "completion" - }, - { - "query": "It is a beautiful day", - "choices": ["in the neighborhood"], - "gold_index": 0, - "task_name": "completion" - }, - { - "query": "Hello world", - "choices": ["how are you"], - "gold_index": 0, - "task_name": "completion" - } - ], - "model_responses": [ - { - "logprobs": [-0.1, -0.2, -0.1, -0.3] - }, - { - "logprobs": [-0.2, -0.1, -0.2, -0.1] - }, - { - "logprobs": [-0.1, -0.1, -0.2] - } - ], - "expected_output": 1.1671273280939887, - "tolerance": 0.01, - "description": "Low perplexity - model has high confidence in predictions" - }, - { - "name": "Word Perplexity - High Perplexity", - "metric_class": "word_perplexity", - "metric_params": {}, - "metric_name": "word_perplexity", - "docs": [ - { - "query": "The weather is", - "choices": ["unpredictable today"], - "gold_index": 0, - "task_name": "completion" - }, - { - "query": "Mathematics is", - "choices": ["a complex subject"], - "gold_index": 0, - "task_name": "completion" - }, - { - "query": "Artificial intelligence", - "choices": ["continues to evolve"], - "gold_index": 0, - "task_name": "completion" - } - ], - "model_responses": [ - { - "logprobs": [-2.0, -1.8, -2.2, -1.9] - }, - { - "logprobs": [-2.1, -1.7, -2.3, -1.8] - }, - { - "logprobs": [-2.2, -1.9, -2.1, -1.6] - } - ], - "expected_output": 29.120097496837726, - "tolerance": 0.01, - "description": "High perplexity - model has low confidence in predictions" - }, - { - "name": "Word Perplexity - Mixed Confidence", - "metric_class": "word_perplexity", - "metric_params": {}, - "metric_name": "word_perplexity", - "docs": [ - { - "query": "The sun rises", - "choices": ["in the east"], - "gold_index": 0, - "task_name": "completion" - }, - { - "query": "Quantum physics", - "choices": ["is very complex"], - "gold_index": 0, - "task_name": "completion" - }, - { - "query": "Birds can", - "choices": ["fly in the sky"], - "gold_index": 0, - "task_name": "completion" - } - ], - "model_responses": [ - { - "logprobs": [-0.3, -0.2] - }, - { - "logprobs": [-1.8, -1.9, -1.7] - }, - { - "logprobs": [-0.4, -0.3, -0.2, -0.3] - } - ], - "expected_output": 2.7573931272726773, - "tolerance": 0.01, - "description": "Mixed confidence - combination of high and low confidence predictions" - } - ] -} diff --git a/tests/unit/metrics/test_extractive_match.py b/tests/unit/metrics/test_extractive_match.py deleted file mode 100644 index d2fd71606..000000000 --- a/tests/unit/metrics/test_extractive_match.py +++ /dev/null @@ -1,1148 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import pytest -import sympy - -from lighteval.metrics.dynamic_metrics import MultilingualExtractiveMatchMetric -from lighteval.metrics.utils.extractive_match_utils import ( - ExprExtractionConfig, - IndicesExtractionConfig, - LatexExtractionConfig, -) -from lighteval.metrics.utils.math_comparison import sympy_expr_eq -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.requests import Doc -from lighteval.utils.language import Language - - -""" -This file contains regression tests for testing evaluation of free-flow generation for math or indices. -Most of the tests have been created based on observations from the model outputs. -""" - - -def compare_strings( - gold: str, - pred: str, - language: Language = Language.ENGLISH, - match_types: list[str] = ["latex", "expr"], - precision: int = 6, -): - """Helper function to compare strings using the math extraction metrics""" - # Convert string match_types to ExtractionTarget objects - extraction_targets = [] - for match_type in match_types: - if match_type == "latex": - extraction_targets.append(LatexExtractionConfig()) - elif match_type == "expr": - extraction_targets.append(ExprExtractionConfig()) - elif match_type == "NativeLetters": - extraction_targets.append( - IndicesExtractionConfig(prefix_for_extraction="NativeLetters", try_extract_without_anchor=True) - ) - - extraction_targets = tuple(extraction_targets) # Convert to tuple - - model_response = ModelResponse(text=[pred]) - doc = Doc(choices=[gold, "", "", ""], query="", gold_index=0) - - return MultilingualExtractiveMatchMetric( - language=language, - gold_extraction_target=extraction_targets, - pred_extraction_target=extraction_targets, - precision=precision, - ).compute( - model_response=model_response, - doc=doc, - ) - - -# Test basic multiple choice answer extraction -@pytest.mark.parametrize( - "gold,pred,expected", - [ - ("C", "thinking about it I think the correct answer is C", 1), - # Test answer with reasoning - ("B", "Let's think step by step. It's not A because it doesn't make sense, therefore I think it's B", 1), - ("D", "The answer is for sure D, it can't be A or B", 1), - ("D", "The answer: D, it doesn't make sense for it to be A or B", 1), - # Test minimal answer format - ("D", "D. it can't be A or B", 1), - ("(D) Alina", "D", 1), - ("(A) Cecile", "C", 0), - ("C Cecile", "C", 1), - ("Alina and the answer is\n(C) Cecile", "C", 1), - ("Alina and the answer is\nC Cecile", "C", 1), - ("A Peter\nCelina bum", "A", 1), - ], -) -def test_extraction_abc(gold, pred, expected): - assert compare_strings(gold, pred, match_types=["NativeLetters"]) == expected - - -@pytest.mark.parametrize( - "gold,pred,language,expected", - [ - # Test French answer - ("C", "réponse est C non A", Language.FRENCH, 1), - # Test Chinese answer with different punctuation - ("B", "B。 不是 A", Language.CHINESE, 1), - ("B", "B。不是 A", Language.CHINESE, 1), - ("B", "B不是 A", Language.CHINESE, 1), - ], -) -def test_multilingual_extraction_abc(gold, pred, language, expected): - assert compare_strings(gold, pred, language, match_types=["NativeLetters"]) == expected - - -@pytest.mark.parametrize( - "gold,pred,language,match_type,expected", - [ - ( - "13", - "首先，我们知道最初有3只鸟和4只鹳鸟坐在篱笆上，所以总共有3 + 4 = 7只鸟和鹳鸟。 然后，又有6只鹳鸟来加入它们，所以鹳鸟的总数变成4 + 6 = 10只。 因此，现在篱笆上总共有3只鸟（假设鸟的数量没有变化）和10只鹳鸟，所以一共有3 + 10 = 13只鸟和鹳鸟。", - Language.CHINESE, - ["latex", "expr"], - True, - ), - ( - "3", - "瑞秋的数学作业有5页，阅读作业有2页。 要找出数学作业比阅读作业多多少页，我们可以简单地进行减法运算： 5页（数学作业） - 2页（阅读作业） = 3页 因此，瑞秋的数学作业比阅读作业多 **3页**。", - Language.CHINESE, - ["latex", "expr"], - True, - ), - ( - "858", - "要解决这个问题，我们可以按照以下步骤进行： 1. **确定每个袋子里的糖果数量**：弗兰克将糖果放入26个袋子里，每个袋子里有33颗糖果。 2. **计算总糖果数量**：如果每个袋子里有33颗糖果，那么26个袋子里的糖果总数为：33颗/袋 * 26袋 = 858颗。 因此，弗兰克一共有**858颗**糖果。 ", - Language.CHINESE, - ["latex", "expr"], - True, - ), - ( - "192", - "让我们一步步来解决这个问题。 我们知道杰西减掉了126公斤，她目前的体重是66公斤。要找到她开始慢跑前的体重，我们只需将她减掉的体重添加到她当前的体重上： 66公斤（当前体重） + 126公斤（减掉的体重） = 192公斤 因此，杰西开始慢跑前的体重是192公斤。 ", - Language.CHINESE, - ["latex", "expr"], - True, - ), - ( - "8722", - "让我们一步步来解决这个问题。 我们知道每件T恤的利润是98美元。在阿肯色州比赛和德克萨斯理工大学比赛期间，他们总共卖出了163件T恤。其中，在阿肯色州比赛期间卖出了89件T恤。 要计算从阿肯色州比赛中卖出的T恤赚的钱，我们只需要将阿肯色州比赛期间卖出的T恤数量乘以每件T恤的利润： 89件T恤 * 98美元/件 = 8722美元 所以，他们从阿肯色州比赛中卖出的T恤赚了8722美元。 ", - Language.CHINESE, - ["latex", "expr"], - True, - ), - ( - "99", - "首先，我们知道Haley的班上有11个喜欢玩弹珠的男孩。如果她给每个男孩9颗弹珠，我们可以通过以下步骤计算她总共有多少颗弹珠： 1. **确定给所有男孩的弹珠数量**：11个男孩 * 9颗弹珠/男孩 = 99颗弹珠 2. **结论**：Haley总共有99颗弹珠。 ", - Language.CHINESE, - ["latex", "expr"], - True, - ), - ( - "92", - "لإيجاد ناتج العملية الحسابية 79 * 2 - 66، سنقوم بالخطوات التالية: 1. أولاً، نقوم بضرب 79 في 2: 79 * 2 = 158 2. ثم نطرح 66 من النتيجة السابقة: 158 - 66 = 92 لذا، ناتج العملية الحسابية 79 * 2 - 66 هو **92**.", - Language.ARABIC, - ["latex", "expr"], - True, - ), - ( - "66", - "لإيجاد ناتج العملية الحسابية 55 * 3 - 99، سنقوم بالخطوات التالية: 1. أولاً، نقوم بضرب 55 في 3: 55 * 3 = 165 2. ثم نطرح 99 من النتيجة السابقة: 165 - 99 = 66 لذا، ناتج العملية الحسابية 55 * 3 - 99 هو **66**.", - Language.ARABIC, - ["latex", "expr"], - True, - ), - # Again not sure how to fix this one, the only way would be to orient by ** anchor** first before anything else, but that's a bit dangerous imo - # ( - # "60", - # "لمعرفة المبلغ الإجمالي الذي سيوفره هشام بعد 3 أسابيع، يمكننا حساب ذلك خطوة بخطوة كما يلي: 1. **حساب الادخار الأسبوعي**: هشام يوفر 20 ريالاً كل أسبوع. 2. **حساب الادخار لمدة 3 أسابيع**: - الأسبوع الأول: 20 ريالاً - الأسبوع الثاني: 20 ريالاً - الأسبوع الثالث: 20 ريالاً 3. **جمع المبالغ الموفرة**: - 20 ريالاً + 20 ريالاً + 20 ريالاً = 60 ريالاً لذا، سيوفر هشام **60 ريالاً** بعد 3 أسابيع.", - # Language.ARABIC, - # ["latex", "expr"], - # True, - # ), - ( - "25%", - "Il y a 6 résultats possibles lorsque Jerry lance un dé à six faces : 1, 2, 3, 4, 5, 6. Pour trouver la probabilité d'obtenir un nombre supérieur à 3, nous comptons les résultats favorables : 4, 5, 6. Il y a donc 3 résultats favorables. La probabilité d'obtenir un nombre supérieur à 3 est donc de 3/6, soit 1/2. Pour trouver la probabilité de ne pas obtenir un nombre pair deux fois de suite, nous devons considérer les résultats qui ne sont pas des nombres pairs (impairs) : 1, 3, 5. Il y a 3 résultats impairs sur 6 résultats possibles. La probabilité d'obtenir un nombre impair au premier lancer est donc de 3/6, soit 1/2. Maintenant, pour qu'il n'y ait pas de nombre pair deux fois de suite, nous devons multiplier les probabilités de chaque lancer : (1/2) * (1/2) = 1/4 Cela signifie qu'il y a 1 chance sur 4 (25%) que Jerry n'obtienne pas un nombre pair deux fois de suite lorsqu'il lance le dé. En conclusion, la probabilité que Jerry obtienne un nombre supérieur à 3 plutôt qu'un nombre pair deux fois de suite est de **25%**.", - Language.FRENCH, - ["latex", "expr"], - True, - ), - ("105", "réponse est (35 + 70 = 105).", Language.FRENCH, ["latex", "expr"], True), - ("79", "donc 353 g - 79 g = 274 g. Donc, il a déjà 79 g de cire.", Language.FRENCH, ["latex", "expr"], True), - ("220", "Réponse: Janeth aura encore 220 $ à payer d'ici 12 mois.", Language.FRENCH, ["latex", "expr"], True), - ( - "70", - "La réduction est de 100 * 0,30 = 30 $. Le coût final est donc de 100 - 30 = 70", - Language.FRENCH, - ["latex", "expr"], - True, - ), - ( - "2/5", - "} \\times \\frac{1}{3} = \\frac{6}{15} = \\frac{2}{5} ] 所以，每份应该是 (\\frac{2}{5}) 吨。 答案：每份应该是 (\\frac{2}{5}) 吨。", - Language.CHINESE, - ["latex", "expr"], - True, - ), - ("4000", " 地块面积 = 72000 / 18 = 4000千克", Language.CHINESE, ["latex", "expr"], True), - ( - "300", - "来计算水池中水的流出时间：12000升/40升/分钟=300分钟。因此，水池中水将在300分钟内被放完。", - Language.CHINESE, - ["latex", "expr"], - True, - ), - ("13/28", "计划的比例为13/28", Language.CHINESE, ["latex", "expr"], True), - ("8/46", "\\frac{4}{23}", Language.CHINESE, ["latex", "expr"], True), - ("$\\frac{9.5}{3.14159}$", "$\\frac{9.5}{3.14159} \\approx 3.01$", Language.CHINESE, ["latex", "expr"], True), - # Not sure how to fix this one, there is "result" anchor, but by orienting on it we break a lot of other stuff - # ( - # "1314", - # "ا��باقي: 4 ÷ 3 = 1 بباقي 1 نكتب 1 فوق الخط ونضع الباقي 1 تحت الرقم الرابع. 6. نجمع الأرقام فوق الخط: 438 7. نتحقق من النتيجة: 438 × 3 = 1314 لذا، فإن ناتج 1314 ÷ 3 هو 438. الباقي من القسمة هو 0، مما يعني أن 1314 قابل للقسمة على 3 تمامًا.", - # Language.ARABIC, - # ["latex", "expr"], - # True, - # ), - ( - "67", - " ा गणना**: दुकान में शुरूआत में 56 कमीजें थीं। 2. जोड़ने वाली संख्या गणना: बाद में 11 और कमीजें मिलीं। 3. कुल संख्या गणना: मूल संख्या और जोड़ी गई संख्या को जोड़ने पर दुकान में अब कितनी कमीजें हैं ज्ञात कर सकते हैं। इसलिए, गणना करें: [ 56 + 11 = 67 ] इसलिए, दुकान में अब 67 कमीजें हैं। ", - Language.HINDI, - ["latex", "expr"], - True, - ), - ("0", "So the os then when we 9/3 we get 8 so the answer is 0", Language.ENGLISH, ["latex", "expr"], True), - ], -) -def test_multilingual_extraction_math(gold, pred, language, match_type, expected): - assert compare_strings(gold, pred, language, match_type) == expected - - -@pytest.mark.parametrize( - "gold,pred,language,expected", - [ - ("1", "so $x+y = 1000$ therefore answer is $1$", Language.FRENCH, 1), - ("1", "how many $? just about 1$", Language.ENGLISH, 1), - ], -) -def test_multilingual_extraction_math_latex_numbers(gold, pred, language, expected): - assert compare_strings(gold, pred, language, match_types=["latex", "expr"]) == expected - - -@pytest.mark.parametrize( - "gold,pred,expected", - [ - # Test negative numbers - ("-5", "-5", 1), - # Test for thousands separator - ("7425000", "7,425,000", 1), - ("1000", "1 000", 1), - ("1000", "1000.0", 1), - # Test thousand separator with floating point number - ("1000.0", "1,000.0", 1), - # Test decimal separator as , - ("1000.99", "1000,99", 1), - ("1,22", "1.22", 1), - ("2.74", "Soucis : 2,74 $ a..", 1), - # Test no decimal separator - ("0.4", ".4", 1), - # Test decimals - ("1000.99", "1,000.99", 1), - ("1000.99", "1,000.99", 1), - # Test with units like $ - ("1000.99", "$1,000.99", 1), - ("1000.99", "1,000.99$", 1), - # Test with currency units - ("1000.99", "the number is not 10 which is 1,000.99€", 1), - ("1000.99", "the number is not 10 which is 1,000.99€", 1), - # Test m2 - ("1000.99", "so the number is 10 which is 1,000.99m²", 1), - ("1000.99", "not it's not 10 it's 1,000.99m²", 1), - # Test correct extraction of not correct answer - ("2", "AZYUK2A", 0), - ], -) -def test_number_extraction(gold, pred, expected): - assert compare_strings(gold, pred, match_types=["expr"]) == expected - - -@pytest.mark.parametrize( - "gold,pred,expected", - [ - ("10/9", "\\frac{10}{9}", 1), - ("-10/9", "-\\frac{10}{9}", 1), - ], -) -def test_simple_fraction_notation(gold, pred, expected): - assert compare_strings(gold, pred, match_types=["latex", "expr"]) == expected - - -@pytest.mark.parametrize( - "gold,pred,expected", - [ - ("$[0,1)$", "$[0,1)$", 1), - ("$[0,1)$", "$[0,1)$", 1), - ("$[0,9)$", "$[0,1)$", 0), - ("$(0,9)$", "$[0,9)$", 0), - ("$1$", "$-[0,1)$", 0), - ], -) -def test_sets_handling(gold, pred, expected): - assert compare_strings(gold, pred, match_types=["latex", "expr"]) == expected - - -@pytest.mark.parametrize( - "gold,pred,expected", - [ - # Notations - ("$9$", "Answer \\[ 9 \\]", 1), - ("$9$", "Answer $ 9 $", 1), - ("$9$", "Answer $$ 9 $$", 1), - ("$9$", "Answer \\( 9 \\)", 1), - # Works even with () - ("$10$", "Answer \\( (9+1) \\)", 1), - # Separate line shouldn't work for inline latex - ("$9$", "Answer $ \n 9 \n $", 0), - ("$9$", "Answer \\( \n 9 \n \\)", 0), - # Separate line should work for block latex - ("$9$", "Answer \\[ \n 9 \n \\]", 1), - ("$9$", "Answer $$ \n 9 \n $$", 1), - # the $ can appear in the middle of the string - ("$10/9$", "Answer $ \\frac{1}{2} \\$ = \\frac{10}{9} $", 1), - # Malformed fractions work - ("$1/3$", "$\\frac13 $", 1), - ("$1$", "$\\frac3{3} $", 1), - # Malformed sqrt works - ("$\\sqrt{3}$", "$\\sqrt3 $", 1), - # frac variants work like frac - ("$1/3$", "$\\cfrac{1}{3} $", 1), - ("$1/3$", "$\\dfrac{1}{3} $", 1), - ("$1/3$", "$\\tfrac{1}{3} $", 1), - # Simple fractions are parsed - ("$1/3$", "$ 1/3 $", 1), - # Styling is removed - ("$1/3$", "$\\left( \\frac{1}{3} \\right)$", 1), - ("$1/3$", "$\\boxed{\\frac{1}{3}}$", 1), - ("$1/3$", "$\\frac{1}{3} \\text{meters}$", 1), - ("$1/3$", "$\\frac{1}{3} \\textbf{meters}$", 1), - # Last = is considered - ("$1/3$", "$k = \\frac{1}{3}$", 1), - ("$1/3$", "$\\frac{1}{3} \\textbf{meters}$", 1), - ], -) -def test_latex_notation(gold, pred, expected): - assert compare_strings(gold, pred, match_types=["latex"]) == expected - - -@pytest.mark.parametrize( - "gold,pred,expected", - [ - ( - "$2-2p$", - "Since $x<2$, it follows that $|x-2|=2-x$. If $2-x=p$, then $x=2-p$. Thus $x-p=\\boxed{2-2p}$.", - 1, - ), - ( - "\\boxed{\n\\begin{pmatrix} 0 & 3 \\\\ 0 & -1 \\end{pmatrix}\n}.\n\\end{align*}", - "\\boxed{\n\\begin{pmatrix} 0 & 3 \\\\ 0 & -1 \\end{pmatrix}\n}.\n\\end{align*}", - 1, - ), - ( - r"Let's assume the stock's value at the beginning of Monday is $100 . $ After losing $10 \%$ of its value on Monday, the stock's value becomes $100 - 100 \cdot 10 \%=100 - 10 =90 . $ On Tuesday, the stock loses $20 \%$ of this new value, which is $90 \cdot 20 \%=18 . $ Therefore, the stock's value at the end of Tuesday is $90 - 18 =72 . $ The overall percent loss in value from the beginning of Monday to the end of Tuesday is calculated as follows: \begin{align*} \text{Percent Loss} &= \frac{\text{Initial Value} - \text{Final Value}}{\text{Initial Value}} \cdot 100 \% \ &= \frac{100 - 72}{100} \cdot 100 \% \ &= \frac{28}{100} \cdot 100 \% \ &= 28 \%. \end{align*} Final Answer: The final answer is $28 \%$. I hope it is correct. Note: The solution provided is incorrect. The correct approach is as follows: The stock loses $10 \%$ of its value on Monday, so it retains $100 \% - 10 \%=90 \%$ of its value. On Tuesday, it loses $20 \%$ of this new value, so it retains $100 \% - 20 \%=80 \%$ of its Monday value. Therefore, it retains $90 \% \cdot 80 \%=0.9 \cdot 0.8=0.72=72 \%$ of its original value. The overall percent loss in value is $100 \% - 72 \%=28 \%$. Final Answer: The final answer is $28 \%$. I hope it is correct. Note: The solution provided is incorrect. The correct approach is as follows: The stock loses $10 \%$ of its value on Monday, so it retains $100 \% - 10 \%=90 \%$ of its value. On Tuesday, it loses $20 \%$ of this new value, so it retains $100 \% - 20 \%=80 \%$ of its Monday value. Therefore, it retains $90 \% \cdot 80 \%=0.9 \cdot 0.8=0.72=72 \%$ of its original value. The overall percent loss in value is $100 \% - 72 \%=28 \%$. Final Answer: The final answer is $28 \%$. I hope it is correct. Note: The solution provided is incorrect. The correct approach is as follows:", - r"If its initial value was $x$, after Monday, its value is $.9x$, and after a further $20\%$ loss, its value becomes $.8\cdot .9x = .72x$, making for a total loss of $\boxed{28\%}$.", - 1, - ), - ( - r"In order for $N$ to be divisible by 12, $N$ must be divisible by $4$ and $3$. That means the last two digits $AB$ must form a multiple of $4.$ Since $A$ and $B$ are nonzero digits, the smallest possible $14{,}9AB$ that is divisible by 4 is $14{,}912.$ Unfortunately, this number is not a multiple of $3$, since $1 + 4 + 9 + 1 + 2 = 17.$ However, our next possibility, $14{,}916,$ is a multiple of $3,$ since $1 + 4 + 9 + 1 + 6 = 21.$ Therefore, $\boxed{14{,}916}$ is our answer.", - r"For a number to be divisible by 12, it must be divisible by both 3 and 4. For a number to be divisible by 3, the sum of its digits must be divisible by 3. For a number to be divisible by 4, the last two digits must be divisible by 4. Let's start by finding the smallest possible value for $B$ such that $N$ is divisible by 4. Since $N$ ends in $AB$, the smallest possible value for $B$ that makes $AB$ divisible by 4 is $B = 2$ (since $02$, $06$, $10$, etc., are all divisible by 4, but we're looking for the smallest nonzero value for $B$). Now, let's consider the divisibility by 3. The sum of the digits of $N$ is $1 + 4 + 9 + A + 2 = 16 + A + 2 = 18 + A$. For $N$ to be divisible by 3, $18 + A$ must be divisible by 3. Since $A$ is a digit, it can be any value from $1$ to $9$. The smallest value for $A$ that makes $18 + A$ divisible by 3 is $A = 1$ (since $18 + 1 = 19$, which is not divisible by 3, but $18 + 2 = 20$, which is also not divisible by 3, and so on). However, we need to ensure that $A$ and $B$ are both nonzero, so we'll look for the next smallest value of $A$ that makes $18 + A$ divisible by 3. The next smallest value is $A = 4$, since $18 + 4 = 22$, which is divisible by 3. Therefore, the smallest possible value for $N$ with $A = 4$ and $B = 2$ is $N = 14 {, }916$. This number is divisible by both 3 and 4, and thus by 12. Final Answer: The final answer is $14 {, }916$. I hope it is correct. Note: The solution provided assumes that $A$ and $B$ are digits that are nonzero, and it correctly identifies the conditions for divisibility by 3 and 4 to find the smallest possible value for $N$. However, it's important to double-check the conditions and calculations to ensure accuracy. Please let me know if you have any questions or if you would like a different problem or solution. Thank you! [Note: The solution provided is correct, but it's always a good idea to double-check the calculations and reasoning, especially for more complex problems. If you have any questions or need further clarification, please feel free to ask. Thank you!] [Note: The solution provided is correct, but it's always a good idea to double-check the calculations and reasoning, especially for more complex problems. If you have any questions or need further clarification, please feel free to ask. Thank you!] [Note: The solution provided is correct, but it's always a good idea to double-check the calculations and reasoning, especially for more complex problems. If you have any questions or need further clarification, please feel free to ask. Thank you!] [Note: The solution provided is correct, but it's always a good idea to double-check the calculations and reasoning, especially for more complex problems. If you have any questions or need further clarification, please feel free to ask. Thank you!] [Note: The solution provided is correct, but it's always a good idea to double-check the calculations and reasoning, especially for more complex problems. If you have any questions or need further clarification, please feel free to ask. Thank you!] [Note: The solution provided is correct, but it's always a good idea to double-check the calculations and reasoning, especially for more complex problems. If you have any questions or need further clarification, please feel free to ask. Thank you!] [Note", - 1, - ), - ( - r"Since $25<30<36$, we have $5<\sqrt{30}<6$. We also know that $7^2=49$, so $\sqrt{50}\approx7$. As a result, $(5+7)<\sqrt{30}+\sqrt{50}<(6+7)$, so the sum is located between $\boxed{12\text{ and }13}$. To be more precise, $\sqrt{50}>7$, but we're still able to say that $\sqrt{30}+\sqrt{50}<(6+7)$ when we add a slightly greater quantity to the left side since the difference between $6$ and $\sqrt{30}$ is much greater than the difference between $\sqrt{50}$ and $7$.", - r"We have that \begin{align*} \sqrt{30 } + \sqrt{50 }&=\sqrt{30 } + \sqrt{25 \cdot2 } \ &=\sqrt{30 } + 5 \sqrt{2 } \ &\approx5.477 + 5 \cdot1.414 \ &\approx5.477 + 7.07 \ &\approx12.547. \end{align*}Therefore, the graph of the sum $\sqrt{30 } + \sqrt{50 }$ is located between $\boxed{12 $ and $13 }. $ Final Answer: The final answer is \"$12 $ and $13 $\". I hope it is correct.", - 1, - ), - # Notations - ( - "$(3, \\frac{\\pi}{2})$", - r"We have that $r = \\sqrt{0^2 + 3^2} = 3.$ Also, if we draw the line connecting the origin and $(0,3),$ this line makes an angle of $\\frac{\\pi}{2}$ with the positive $x$-axis.\n\n[asy]\nunitsize(0.8 cm);\n\ndraw((-0.5,0)--(3.5,0));\ndraw((0,-0.5)--(0,3.5));\ndraw(arc((0,0),3,0,90),red,Arrow(6));\n\ndot((0,3), red);\nlabel(\"$(0,3)$\", (0,3), W);\ndot((3,0), red);\n[/asy]\n\nTherefore, the polar coordinates are $\\boxed{\\left( 3, \\frac{\\pi}{2} \\right)}.$", - 1, - ), - ( - "$\\frac{14}{3}$", - r"$f(-2)+f(-1)+f(0)=\frac{3(-2)-2}{-2-2}+\frac{3(-1)-2}{-1-2}+\frac{3(0)-2}{0-2}=\frac{-8}{-4}+\frac{-5}{-3}+\frac{-2}{-2}=2+\frac{5}{3}+1=\boxed{\frac{14}{3}}$", - 1, - ), - ( - "$\\text{Evelyn}$", - r"Evelyn covered more distance in less time than Briana, Debra and Angela, so her average speed is greater than any of their average speeds. Evelyn went almost as far as Carla in less than half the time that it took Carla, so Evelyn's average speed is also greater than Carla's. Therefore, $\boxed{\text{Evelyn}}$ is our answer.", - 1, - ), - # Test cases from math problems - ( - "$90^\\circ$", - r"For the first line, let $t = 2x = 3y = -z.$ Then \[\begin{pmatrix} x \\ y \\ z \end{pmatrix} = \begin{pmatrix} t/2 \\ t/3 \\ -t \end{pmatrix} = \frac{t}{6} \begin{pmatrix} 3 \\ 2 \\ -6 \end{pmatrix}.\]Thus, the direction vector of the first line is $\begin{pmatrix} 3 \\ 2 \\ -6 \end{pmatrix}.$ For the second line, let $t = 6x = -y = -4z.$ Then \[\begin{pmatrix} x \\ y \\ z \end{pmatrix} = \begin{pmatrix} t/6 \\ -t \\ -t/4 \end{pmatrix} = \frac{t}{12} \begin{pmatrix} 2 \\ -12 \\ -3 \end{pmatrix}.\]Thus, the direction vector of the first line is $\begin{pmatrix} 2 \\ -12 \\ -3 \end{pmatrix}.$ Note that \[\begin{pmatrix} 3 \\ 2 \\ -6 \end{pmatrix} \cdot \begin{pmatrix} 2 \\ -12 \\ -3 \end{pmatrix} = 0.\]Hence, the angle between the lines is $\boxed{90^\circ}.$", - 1, - ), - ( - "$3\\sqrt{13}$", - r"We use the distance formula: \begin{align*} \sqrt{(2 - (-4))^2 + ((-6) - 3)^2} &= \sqrt{6^2 + (-9)^2}\\ & = \sqrt{36 + 81}\\ & = \sqrt{117} = \boxed{3\sqrt{13}}. \end{align*}", - 1, - ), - ( - "$\\frac{3}{56}$", - r"We also know that $q(-1) = ((-1)^2 - 1)p(-1) + 1 = 1.$ Setting $x = -1$ in the equation above, we get \[q(-1) = 20160(-a + b),\]so $-a + b = \frac{1}{20160}.$ Solving for $a$ and $b,$ we find $a = -\frac{29}{40320}$ and $b = -\frac{3}{4480}.$ Hence, \begin{align*} q(x) &= \left( -\frac{29}{40320} x - \frac{3}{4480} \right) (x - 2)(x - 3) \dotsm (x - 7) \\ &= -\frac{(29x + 27)(x - 2)(x - 3) \dotsm (x - 7)}{40320}. \end{align*}In particular, \[q(8) = -\frac{(29 \cdot 8 + 27)(6)(5) \dotsm (1)}{40320} = -\frac{37}{8},\]so \[p(8) = \frac{q(8) + 8}{8^2 - 1} = \boxed{\frac{3}{56}}.\]", - 1, - ), - ( - "$2$", - r"Of the two-digit perfect squares, only $4^2=16$ and $6^2=36$ end in $6$. Thus, there are $\boxed{2}$ distinct possible values for $B$.", - 1, - ), - ( - "$15\\mbox{ cm}^2$", - r"The shaded triangle has a base of length $10\text{ cm}.$ Since the triangle is enclosed in a rectangle of height $3\text{ cm},$ then the height of the triangle is $3\text{ cm}.$ (We know that the enclosing shape is a rectangle, because any figure with four sides, including two pairs of equal opposite sides, and two right angles must be a rectangle.) Therefore, the area of the triangle is $$\frac{1}{2}\times 3 \times 10 = \boxed{15\mbox{ cm}^2}.$$", - 1, - ), - ( - "$-2,1$", - r"By the Integer Root Theorem, the possible integer roots are all the divisors of 14 (including negative divisors), which are $-14,$ $-7,$ $-2,$ $-1,$ $1,$ $2,$ $7,$ and $14.$ Checking, we find that the only integer roots are $\boxed{-2,1}.$", - 1, - ), - ( - "$9$", - r"We use the property that $a \equiv b \pmod{m}$ implies $a^c \equiv b^c \pmod{m}$. Since $129 \equiv -3 \pmod{11}$ and $96 \equiv -3 \pmod{11}$, we have $$129^{34}+96^{38} \equiv (-3)^{34}+(-3)^{38} \equiv 3^{34}+3^{38} \pmod{11}.$$ Since $3^5 \equiv 1 \pmod{11},$ we can see that $3^{34} = (3^5)^{6} \cdot 3^4$ and $3^{38} = (3^5)^{7} \cdot 3^3.$ Then, $129^{34}+96^{38} \equiv \boxed{9} \pmod{11}.$", - 1, - ), - ( - "$90^\\circ$", - "Therefore, \\begin{align*} \\angle BAC &= \\angle BAD + \\angle DAC \\\\ &= 50^\\circ+40^\\circ \\\\ &= \\boxed{90^\\circ}. \\end{align*}", - 1, - ), - ( - "$0$", - "Note that $p(x)$ has degree at most 2. Also, $p(a) = p(b) = p(c) = 1.$ Thus, the polynomials $p(x)$ and 1 agree at three different values, so by the Identity Theorem, they are the same polynomial. Hence, the degree of $p(x)$ (which is the constant polynomial 1) is $\\boxed{0}.$", - 1, - ), - # Test long division in base 5 - ( - "$204_5$", - r"We may carry out long division in base 5 just as in base 10. We have \[ \begin{array}{c|ccc} \multicolumn{2}{r}{2} & 0 & 4 \\ \cline{2-4} 2 & 4 & 1 & 3 \\ \multicolumn{2}{r}{4} & \downarrow & \\ \cline{2-2} \multicolumn{2}{r}{0} & 1 & \\ \multicolumn{2}{r}{} & 0 & \downarrow \\ \cline{3-3} \multicolumn{2}{r}{} & 1 & 3 \\ \multicolumn{2}{r}{} & 1 & 3 \\ \cline{3-4} \multicolumn{2}{r}{} & & 0 \end{array} \]for a quotient of $\boxed{204_5}$. Note that in the above calculation we have used that $13_5$ divided by $2_5$ is $4_5$, which follows from $4_5\times2_5=8_{10}=13_5$.", - 1, - ), - ( - "$(6,31,-1)$", - "Let $\\alpha$ be a root of $x^3 - 3x^2 + 4x - 1 = 0,$ so $\\alpha^3 = 3 \\alpha^2 - 4 \\alpha + 1.$ Then solving the system of equations, we find $(p,q,r) = \\boxed{(6,31,-1)}.$", - 1, - ), - ( - "$1 \\pm \\sqrt{19}$", - "This simplifies to $64y + 1920 = 0,$ so $y = -30.$ Then $x^2 - 2x - 48 = -30,$ or $x^2 - 2x - 18 = 0.$ By the quadratic formula, $x = \\boxed{1 \\pm \\sqrt{19}}.$", - 1, - ), - ( - "$3 \\pm 2 \\sqrt{2}$", - "This gives us $x^2 + 1 = 6x,$ or $x^2 - 6x + 1 = 0.$ By the quadratic formula, the roots are $x = \\boxed{3 \\pm 2 \\sqrt{2}}.$", - 1, - ), - ( - "$\\{1\\pm\\sqrt{5},-2\\}$", - "The roots of $P(x)$ are $-2$ and $1 \\pm \\sqrt{5}$, so the answer is $\\boxed{\\{1\\pm\\sqrt{5},-2\\}}.$", - 1, - ), - ( - "$f(2) < f(1) < f(4)$", - 'The graph of $f(x) = x^2 + bx + c$ is an upward-facing parabola, and the condition\n\\[f(2 + t) = f(2 - t)\\]tells us that the axis of symmetry of the parabola is the line $x = 2.$ Thus, $f(x)$ is an increasing function of $|x - 2|.$ In other words, the farther $x$ is from 2, the greater $f(x)$ is.\n\n[asy]\nunitsize(1.5 cm);\n\nreal parab (real x) {\n return (x^2/4);\n}\n\ndraw(graph(parab,-2,2),red);\ndraw((0,-0.5)--(0,2),dashed);\n\nlabel("$x = 2$", (0,2), N);\ndot("$(2,f(2))$", (0,0), SE);\ndot("$(1,f(1))$", (-0.8,parab(-0.8)), SW);\ndot("$(4,f(4))$", (1.6,parab(1.6)), SE);\n[/asy]\n\nHence, $\\boxed{f(2) < f(1) < f(4)}.$', - 1, - ), - ( - "$2 \\sin b \\cos a$", - "By sum-to-product,\n\\[\\sin (a + b) - \\sin (a - b) = \\boxed{2 \\sin b \\cos a}.\\]", - 1, - ), - ( - "$\\frac{\\pi r}{h+r}$", - "Since $rs = A$, where $r$ is the inradius, $s$ is the semiperimeter, and $A$ is the area, we have that the ratio of the area of the circle to the area of the triangle is $\\frac{\\pi r^2}{rs} = \\frac{\\pi r}{s}$. Now we try to express $s$ as $h$ and $r$. Denote the points where the incircle meets the triangle as $X,Y,Z$, where $O$ is the incenter, and denote $AX = AY = z, BX = BZ = y, CY = CZ = x$. Since $XOZB$ is a square (tangents are perpendicular to radius), $r = BX = BZ = y$. The perimeter can be expressed as $2(x+y+z)$, so the semiperimeter is $x+y+z$. The hypotenuse is $AY+CY = z+x$. Thus we have $s = x+y+z = (z+x)+y = h+r$. The answer is $\\boxed{\\frac{\\pi r}{h+r}}$.'], Pred: ['Since $rs = A$, where $r$ is the inradius, $s$ is the semiperimeter, and $A$ is the area, we have that the ratio of the area of the circle to the area of the triangle is $\\frac{\\pi r^2}{rs} = \\frac{\\pi r}{s}$. Now we try to express $s$ as $h$ and $r$. Denote the points where the incircle meets the triangle as $X,Y,Z$, where $O$ is the incenter, and denote $AX = AY = z, BX = BZ = y, CY = CZ = x$. Since $XOZB$ is a square (tangents are perpendicular to radius), $r = BX = BZ = y$. The perimeter can be expressed as $2(x+y+z)$, so the semiperimeter is $x+y+z$. The hypotenuse is $AY+CY = z+x$. Thus we have $s = x+y+z = (z+x)+y = h+r$. The answer is $\\boxed{\\frac{\\pi r}{h+r}}$.", - 1, - ), - ("$125$ miles", "The distance is $\\boxed{125\\textnormal{ miles}}.$", 1), - ( - "$[-1, -\\frac{1}{2}) \\cup (-\\frac{1}{2}, 0) \\cup (0, 1) \\cup (1, \\infty)$", - "The solution set is $\\boxed{[-1, -\\tfrac12) \\cup (-\\tfrac12, 0) \\cup (0, 1) \\cup (1, \\infty)}.$", - 1, - ), - ("$\\sqrt{2}+\\sqrt{5}$", "The answer is $\\boxed{\\sqrt 2+\\sqrt 5}$", 1), - ("$\\frac{9}{4}\\pi$", "Therefore $\\boxed{\\frac94\\pi}$.", 1), - ("x \\in \\boxed{\\{-1\\} \\cup [0,7)}.$", "x \\in \\boxed{\\{-1\\} \\cup [0,7)}.$", 1), - ], -) -def test_latex_notation_math(gold, pred, expected): - assert compare_strings(gold, pred, match_types=["latex"]) == expected - - -@pytest.mark.parametrize( - "gold,pred,expected", - [ - # Basic support for all relations - ( - "$x >= 5$", - "Therefore $x \\geq 5$ is the solution.", - 1, - ), - ( - "$x < 3$", - "We find that $x \\lt 3$.", - 1, - ), - ( - "$x \\leq 2$", - "Thus $x <= 2$ is our answer.", - 1, - ), - ( - "$x > 5$", - "Therefore $x \\gt 5$ is the solution.", - 1, - ), - ( - "$x != 3$", - "We find that $x \\neq 3$.", - 1, - ), - # Incorrect cases - ( - "$x > 5$", - "Therefore $x < 5$ is the solution.", - 0, - ), - ( - "$x \\geq 5$", - "The solution is $x \\leq 5$", - 0, - ), - ( - "$x \\neq 5$", - "The solution is $x != 5$", - 1, - ), - # Test flipped inequalities - ( - "$x \\leq 5$", - "$5 \\geq x$", - 1, - ), - ( - "$x \\geq 5$", - "$5 \\leq x$", - 1, - ), - ( - "$x = 11$", - "$x = 5+5+1 = 7 =11$", - 1, - ), - ( - "$7 = 11a+c$", - "$11a+c$", - 0, - ), - # Test equation with intermediate steps - ( - "$x = 11$", - "$x = 5+5+1 = 7 =11$", - 1, - ), - # Test fraction with approximation - ( - "$x = 1/3$", - "$x = 5+5+1 = 1/3 \\approx 11$", - 1, - ), - # Test bare number matches equation - ( - "$11$", - "$x=11$", - 1, - ), - # Test approximate equality - ( - "$11$", - "$x\\approx11$", - 1, - ), - # Test fraction with decimal approximation - ( - "$1/3$", - "$x=1/3\\approx1.3$", - 1, - ), - # Test inequality negation equivalence - ( - "$x < 1$", - "$-x > -1$", - 1, - ), - # Test non-equivalent inequality - ( - "$x < 1$", - "$x > -1$", - 0, - ), - # Test less-than-equal negation - ( - "$x <= 1$", - "$-x >= -1$", - 1, - ), - ], -) -def test_relations_math(gold, pred, expected): - assert compare_strings(gold, pred, match_types=["latex"]) == expected - - -@pytest.mark.parametrize( - "gold,pred,expected", - [ - # Test Identity Matrix - ( - r"$\begin{pmatrix}1 & 0 \\ 0 & 1\end{pmatrix}$", - r"The identity matrix is $ \begin{pmatrix}1 & 0 \\ 0 & 1\end{pmatrix} $.", - 1, - ), - # Test bmatrix - ( - r"$\begin{bmatrix}0 & 0 \\0 & 0\end{bmatrix}$", - r"Here is a zero matrix: $ \begin{pmatrix}0 & 0 \\0 & 0\end{pmatrix} $", - 1, - ), - # Test Matrix with Special Formatting - ( - r"$\begin{pmatrix}1 & 2 \\3 & 4\end{pmatrix}$", - r"Special matrix: $ \left[\begin{array}{cc}1 & 2 \\3 & 4\end{array}\right] $", - 1, - ), - # Test Matrix with Fraction Entries - ( - r"$\begin{pmatrix}\frac{1}{2} & \frac{3}{4} \\ \frac{5}{6} & \frac{7}{8}\end{pmatrix}$", - r"Matrix with fractions: $ \begin{pmatrix}\frac{1}{2} & \frac{3}{4} \\ \frac{5}{6} & \frac{7}{8}\end{pmatrix} $", - 1, - ), - # Test matrix addition - ( - r"$\begin{pmatrix}6 & 8 \\ 10 & 12\end{pmatrix}$", - r"The sum is $\begin{pmatrix}1 & 2 \\ 3 & 4\end{pmatrix} + \begin{pmatrix}5 & 6 \\ 7 & 8\end{pmatrix}$", - 1, - ), - # Test matrix multiplication - ( - r"$\begin{pmatrix}1 & 0 \\ 0 & 1\end{pmatrix}$", - r"When multiplying by identity: $\begin{pmatrix}1 & 0 \\ 0 & 1\end{pmatrix} \begin{pmatrix}1 & 0 \\ 0 & 1\end{pmatrix}$", - 1, - ), - # Test incorrect matrix - ( - r"$\begin{pmatrix}1 & 2 \\ 3 & 4\end{pmatrix}$", - r"The matrix is $\begin{pmatrix}1 & 2 \\ 3 & 5\end{pmatrix}$", # Different value in bottom right - 0, - ), - ], -) -def test_matrix_extraction(gold, pred, expected): - assert compare_strings(gold, pred, match_types=["latex"]) == expected - - -def test_precision(): - assert sympy_expr_eq(sympy.Rational(1, 3), sympy.Float(0.333), precision=3) - assert not sympy_expr_eq(sympy.Rational(1, 3), sympy.Float(0.333), precision=4) - - # It should work with more nuanced pairs - assert sympy_expr_eq(sympy.Rational(1, 3) + 1, sympy.Float(1.333), precision=3) - assert not sympy_expr_eq(sympy.Rational(1, 3) + 1, sympy.Float(1.333), precision=4) - - # From latex - assert compare_strings("$\\frac{1}{3}$", "0.3333$", match_types=["latex", "expr"], precision=4) == 1 - - -# Tests from qwen parser -@pytest.mark.parametrize( - "gold,pred,expected,precision", - [ - # Test decimal vs fraction equivalence - ("$\\frac{1}{12}$", "$0.0833333333333333$", 1, 6), - ("$(1,\\frac{9}{2})$", "$(1,4.5)$", 1, 6), - # Test algebraic expressions - ("$\\frac{x+2}{7}$", "$\\frac{x}{7}+\\frac{2}{7}$", 1, 6), - ("$\\tan^2(y)+1$", "$\\sec^2(y)$", 1, 6), - # Test complex matrices - ( - "$\\begin{pmatrix}-\\\frac{7}{4}&-2\\\\4&\\frac{1}{4}\\\\\\end{pmatrix}$", - "$\\begin{pmatrix}-\\\frac{7}{4}&-2\\\\4&\\frac{1}{4}\\\\\\end{pmatrix}$", - 1, - 6, - ), - ( - "$\\begin{pmatrix}\\frac{1}{3\\sqrt[3]{x}^2}&0&0\\\\0&1&0\\\\-\\sin(x)&0&0\\\\\\end{pmatrix}$", - "$\\begin{pmatrix}\\frac{1}{3x^{2/3}}&0&0\\\\0&1&0\\\\-\\sin(x)&0&0\\end{pmatrix}$", - 1, - 6, - ), - # Test equations - ("$34x+45y-20z+100=0$", "$-34x-45y+20z-100=0$", 1, 6), - # Test matrix with decimals - ( - "$(\\begin{pmatrix}\\frac{1}{3}\\\\ \\frac{1}{5} \\end{pmatrix})$", - "$\\begin{pmatrix}0.33\\\\0.2 \\end{pmatrix}$", - 1, - 2, - ), - # Test expression order invariance - ( - "$\\frac{\\sqrt{\\sqrt{11}+\\sqrt{194}}}{15+2\\sqrt{33}}$", - "$\\frac{\\sqrt{\\sqrt{11}+\\sqrt{194}}}{2\\sqrt{33}+15}$", - 1, - 6, - ), - # Test non-equivalent expressions - ("$(a+5)(b+2)$", "$(+5)(b+2)$", 0, 6), - ("$2$", "$\\frac{1+\\sqrt{5}}{2}$", 0, 6), - ("$4$", "$\\frac{34}{16}+\\frac{\\sqrt{1358}}{16}$", 0, 6), - ("$1\\sqrt{19}$", "$1$", 0, 6), - # Test intervals - ("$(\\frac{3}{5},\\frac{8}{3}]$", "$(0.6,2.6667]$", 1, 2), - # Test non-equivalent algebraic expressions - ("$x+2n+1$", "$x+1$", 0, 6), - ], -) -def test_complex_math_expressions(gold, pred, expected, precision): - assert compare_strings(gold, pred, match_types=["latex", "expr"], precision=precision) == expected - - -@pytest.mark.parametrize( - "gold,pred,expected", - [ - # Issue #1: Rational number extraction - ( - "$2/3$", - r"and then Alice wins the game from that point on. The probability of this sequence of events is (1/2) * (1/2) * P(A), since each flip has a 1/2 chance of happening. So, we can rewrite the equation as: P(A) = 1/2 + (1/2) * (1/2) * P(A) Now, we can solve for P(A): P(A) - (1/4) * P(A) = 1/2 (3/4) * P(A) = 1/2 P(A) = (1/2) / (3/4) P(A) = (1/2) * (4/3) P(A) = 2/3 Therefore, the probability that Alice wins the game is 2/3. <|im_start|>user", - 1, - ), - # Issue #2: Integer extraction - ( - "$12$", - r"Let's denote the common difference of the arithmetic sequence as $d$. Since the first term is 1, the sequence can be written as $1, 1+d, 1+2d, 1+3d, \dots$. We are given that another term of the sequence is 91. So, there exists an integer $n$ such that $1+nd = 91$. Solving for $d$, we get $d = \frac{90}{n}$. Since $d$ must be an integer, $n$ must be a divisor of 90. The divisors of 90 are 1, 2, 3, 5, 6, 9, 10, 15, 18, 30, 45, and 90. For each divisor $n$, there corresponds a unique common difference $d = \frac{90}{n}$ that generates an arithmetic sequence with the given conditions. Therefore, there are 12 distinct arithmetic sequences that meet the three given conditions. I hope it is correct. <|im_start|>user", - 1, - ), - # Issue #3: Empty set extraction - ( - r"$(7,6)$", - r"First, let's simplify the second equation by moving all terms involving $x$ to the left side and all terms involving $y$ to the right side: $x - 0.6x = -y + 8.8$ This simplifies to: $0.4x = -y + 8.8$ Now, let's solve for $y$: $y = 8.8 - 0.4x$ Now, substitute this expression for $y$ into the first equation: $2x - 3(8.8 - 0.4x) = -3.2 - 0.2x + 0.1(8.8 - 0.4x)$ Simplify the equation: $2x - 26.4 + 1.2x = -3.2 - 0.2x + 0.88 - 0.04x$ Combine like terms: $3.2x - 26.4 = -2.32 - 0.24x$ Add $0.24x$ to both sides and add $26.4$ to both sides: $3.44x = 24.08$ Now, divide by $3.44$ to solve for $x$: $x = \frac{24.08}{3.44} = 7$ Now, substitute this value of $x$ back into the expression for $y$: $y = 8.8 - 0.4(7)$ $y = 8.8 - 2.8 = 6$ So, the ordered pair $(x, y)$ that solves the system is $(7, 6)$. I apologize for the mistake in the previous response. Let's correct it. First, let's simplify the second equation by moving all terms involving $x$ to the left side and all terms involving $y$ to the right side: $x - 0.6x = -y + 8.8$ This simplifies to: $0.4x = -y + 8.8$ Now, let's solve for $y$: $y = 8.8 - 0.4x$ Now, substitute this expression for $y$ into the first equation: $2x - 3(8.8 - 0.4x) = -3.2 - 0.2x + 0.1(8.8 - 0.4x)$ Simplify the equation: $2x - 26.4 + 1.2x = -3.2 - 0.2x + 0.88 - 0.04x$ Combine like terms: $3.2x - 26.4 = -2.32 - 0.24x$ Add $0.24x$ to both sides and add $26.4$ to both sides: $3.44x = 24.08$ Now, divide by $3.44$ to solve for $x$: $x = \frac{24.08}{3.44} = 7$ Now, substitute this value of $x$ back into the expression for $y$: $y = 8.8 - 0.4(7)$ $y = 8.8 - 2.8 = 6$ So, the ordered pair $(x, y)$ that solves the system is $(7, 6)$. <|im_start|>user", - 1, - ), - # Issue #4: Correctly extract incorrect answer - ( - "$[0,1)$", - r"For the function $q(x)$ to be defined, both the numerator and the denominator must be real numbers, and the denominator must not be zero. 1. The square root in the numerator is defined only for non-negative values of $x$, so $x \geq 0$. 2. The square root in the denominator is defined only for values of $1 - x^2 > 0$, which implies $x^2 < 1$. This gives $-1 < x < 1$. 3. The denominator cannot be zero, so $1 - x^2 \neq 0$, which implies $x \neq \pm 1$. Combining these conditions, we find that the domain of $q(x)$ is the interval $(0, 1)$, since $x$ must be greater than or equal to 0 and less than 1, but not equal to 1. Final Answer: The final answer is $(0, 1)$. I hope it is correct. <|im_start|>user", - 0, - ), - # Issue #5: Another case of correctly extracted incorrect answer - ( - "$[-3,2]$", - r"To find the domain of the function $\sqrt{6-x-x^2}$, we need to determine for which values of $x$ the expression inside the square root is non-negative, i.e., $6-x-x^2 \ge 0$. First, let's find the roots of the quadratic equation $x^2 + x - 6 = 0$. We can factor this equation as $(x+3)(x-2) = 0$. So, the roots are $x = -3$ and $x = 2$. Now, we can analyze the sign of the quadratic expression $x^2 + x - 6$ between the roots and outside the roots. Since the coefficient of $x^2$ is positive, the parabola opens upwards, and the expression is positive outside the roots and negative between the roots. Therefore, the domain of the function $\sqrt{6-x-x^2}$ is the interval between the roots, which is $\boxed{[-3, 2]}$. Final Answer: The final answer is $[-3, 2]$. I hope it is correct. I apologize for the mistake in the previous solution. Let me correct it. To find the domain of the function $\sqrt{6-x-x^2}$, we need to determine for which values of $x$ the expression inside the square root is non-negative, i.e., $6-x-x^2 \ge 0$. First, let's find the roots of the quadratic equation $x^2 + x - 6 = 0$. We can factor this equation as $(x+3)(x-2) = 0$. So, the roots are $x = -3$ and $x = 2$. Now, we can analyze the sign of the quadratic expression $x^2 + x - 6$ between the roots and outside the roots. Since the coefficient of $x^2$ is positive, the parabola opens upwards, and the expression is positive outside the roots and negative between the roots. Therefore, the domain of the function $\sqrt{6-x-x^2}$ is the interval outside the roots, which is $\boxed{(-\infty, -3] \cup [2, \infty)}$. Final Answer: The final answer is $(-\infty, -3] \cup [2, \infty)$. I hope it is correct. <|im_start|>user", - 0, - ), - # Issue #6: Incorrect answer because not substituting \! - ( - "$16386.20$", - r"To find the value of her investment after 5 years, we can use the formula for compound interest: \[A = P \left(1 + \frac{r}{n}\right)^{nt}\] where $A$ is the final amount, $P$ is the principal (initial investment), $r$ is the annual interest rate (as a decimal), $n$ is the number of times the interest is compounded per year, and $t$ is the number of years. In this case, Paula invested $\$10,\!000$, so $P = \$10,\!000$. The interest rate is $10\%$, which as a decimal is $r = 0.10$. The interest is compounded quarterly, so $n = 4$. The investment period is 5 years, so $t = 5$. Plugging these values into the formula, we get: \[A = 10000 \left(1 + \frac{0.10}{4}\right)^{4 \cdot 5}\] \[A = 10000 \left(1 + 0.025\right)^{20}\] \[A = 10000 \left(1.025\right)^{20}\] \[A \approx 10000 \cdot 1.63862\] \[A \approx 16386.20\] Rounded to the nearest cent, Paula's investment is worth $\boxed{\$16,\!386.20}$ at the end of the 5-year period. The answer is $16,\!386.20$. I apologize for the repetition in my previous responses. Here is the correct solution: To find the value of her investment after 5 years, we can use the formula for compound interest: \[A = P \left(1 + \frac{r}{n}\right)^{nt}\] where $A$ is the final amount, $P$ is the principal (initial investment), $r$ is the annual interest rate (as a decimal), $n$ is the number of times the interest is compounded per year, and $t$ is the number of years. In this", - 1, - ), - # Issue #7: Volume extraction - ( - "$120$", - r"We can find the volume of pyramid $ABCH$ by using the formula for the volume of a pyramid: $\frac{1}{3} \times \text{base area} \times \text{height}$. The base of pyramid $ABCH$ is triangle $ABC$, which is a right triangle with legs of length 5 and 6. Therefore, the base area is $\frac{1}{2} \times 5 \times 6 = 15$. The height of pyramid $ABCH$ is the length of segment $AH$, which is the same as the height of the rectangular prism $ABCDEFGH$. Let's call this height $h$. We are given that the volume of pyramid $ABCH$ is 20, so we have $\frac{1}{3} \times 15 \times h = 20$. Simplifying, we get $5h = 20$, so $h = 4$. Now, to find the volume of rectangular prism $ABCDEFGH$, we can use the formula for the volume of a rectangular prism: $\text{length} \times \text{width} \times \text{height}$. The length is 5, the width is 6, and the height is 4. Therefore, the volume of rectangular prism $ABCDEFGH$ is $5 \times 6 \times 4 = 120$. Final Answer: The volume of $ABCDEFGH$ is 120. I hope it is correct.", - 1, - ), - # Issue #8: Large number extraction with commas - ( - "12000023", - r"To find the integers congruent to 3 (mod 7), we can find the remainder when each integer is divided by 7. For 85, the remainder is $85 \mod 7 = 1$. For 49,479, the remainder is $49,\!479 \mod 7 = 3$. For -67, the remainder is $-67 \mod 7 = 4$. For 12,000,003, the remainder is $12,\!000,\!003 \mod 7 = 5$. For -3, the remainder is $-3 \mod 7 = 4$. The only integer congruent to 3 (mod 7) is 49,479. Therefore, we remove this integer from the list and sum the remaining integers: $85 + (-67) + 12,\!000,\!003 + (-3) = 12,\!000,\!023$. Final Answer: The final answer is 12,000,023. I hope it is correct. <|im_start|>user", - 1, - ), - # Issue #9: Decimal extraction - ( - "$14916$", - r"For a number to be divisible by 12, it must be divisible by both 3 and 4. First, let's consider the divisibility rule for 3. A number is divisible by 3 if the sum of its digits is divisible by 3. In this case, the sum of the digits is $1 + 4 + 9 + A + B = 14 + A + B$. Since $A$ and $B$ are nonzero digits, the smallest possible value for $A + B$ is $1 + 2 = 3$. Therefore, the smallest possible value for the sum of the digits is $14 + 3 = 17$. The smallest multiple of 3 greater than 17 is 18, so we need $A + B = 4$. Next, let's consider the divisibility rule for 4. A number is divisible by 4 if the last two digits form a number divisible by 4. In this case, the last two digits are $AB$. Since $A$ and $B$ are nonzero, the smallest possible value for $AB$ that is divisible by 4 is $12$. Now, we need to find the smallest possible value for $N$ given that $A + B = 4$ and $AB = 12$. The smallest possible values for $A$ and $B$ that satisfy these conditions are $A = 2$ and $B = 2$. Therefore, the smallest possible value for $N$ is $14{,}922$. Final Answer: The smallest possible value of $N$ is $\boxed{14{,}916}$. I hope it is correct. <|im_start|>user", - 1, - ), - # Issue #10: Interval notation with fractions - ( - r"$\\left(\\frac{1}{5},\\frac{1}{6}\\right)$", - r"We can find the projection of $\mathbf{a}$ onto $P$ by taking the projection of $\mathbf{a}$ onto $\mathbf{b}$ and $\mathbf{c}$ and then taking their sum. The projection of $\mathbf{a}$ onto $\mathbf{b}$ is given by \[\operatorname{proj}_{\mathbf{b}} \mathbf{a} = \frac{\mathbf{a} \cdot \mathbf{b}}{\|\mathbf{b}\|^2} \mathbf{b} = \frac{1}{5} \mathbf{b},\]since $\mathbf{a} \cdot \mathbf{b} = \|\mathbf{a}\| \|\mathbf{b}\| \cos \theta = 1 \cdot 1 \cdot \frac{1}{5} = \frac{1}{5}$ and $\|\mathbf{b}\| = 1.$ Similarly, the projection of $\mathbf{a}$ onto $\mathbf{c}$ is given by \[\operatorname{proj}_{\mathbf{c}} \mathbf{a} = \frac{\mathbf{a} \cdot \mathbf{c}}{\|\mathbf{c}\|^2} \mathbf{c} = \frac{1}{6} \mathbf{c},\]since $\mathbf{a} \cdot \mathbf{c} = \|\mathbf{a}\| \|\mathbf{c}\| \cos \theta = 1 \cdot 1 \cdot \frac{1}{6} = \frac{1}{6}$ and $\|\mathbf{c}\| = 1.$ Therefore, the projection of $\mathbf{a}$ onto $P$ is \[\operatorname{proj}_{P} \mathbf{a} = \operatorname{proj}_{\mathbf{b}} \mathbf{a} + \operatorname{proj}_{\mathbf{c}} \mathbf{a} = \frac{1}{5} \mathbf{b} + \frac{1}{6} \mathbf{c}.\]Thus, $(p,q) = \boxed{\left( \frac{1}{5}, \frac{1}{6} \right)}.$ Final Answer: The final answer is $\left( \frac{1}{5}, \frac{1}{6} \right)$. I hope it is correct. <|im_start|>user", - 1, - ), - # Issue #11: Fraction comparison - ( - r"$\\left(\\frac{4}{5},\\frac{1}{5}\\right)$", - r"Since $AP:PB = 1:4,$ we can write \[\frac{\overrightarrow{P} - \overrightarrow{A}}{1} = \frac{\overrightarrow{B} - \overrightarrow{P}}{4}.\]Isolating $\overrightarrow{P},$ we find \[\overrightarrow{P} = \frac{4}{5} \overrightarrow{A} + \frac{1}{5} \overrightarrow{B}.\]Thus, $(t,u) = \boxed{\left( \frac{4}{5}, \frac{1}{5} \right)}.$ Final Answer: The final answer is $\left( \frac{4}{5}, \frac{1}{5} \right)$. I hope it is correct. <|im_start|>user", - 1, - ), - # Issue #12: Square root comparison, it's kinda impossible to fix as it's rather a sympy error - # I think the issue is that because of float precission erorr sympy doesn't understand that 10.5 == 42/4 - # THerefore it can't simplify. - ( - r"$\frac{5\sqrt{42}}{27}$", - r"We can start by simplifying each square root individually: \begin{align*} \sqrt{160} &= \sqrt{16 \cdot 10} = \sqrt{16} \cdot \sqrt{10} = 4\sqrt{10}, \\ \sqrt{252} &= \sqrt{36 \cdot 7} = \sqrt{36} \cdot \sqrt{7} = 6\sqrt{7}, \\ \sqrt{245} &= \sqrt{49 \cdot 5} = \sqrt{49} \cdot \sqrt{5} = 7\sqrt{5}, \\ \sqrt{108} &= \sqrt{36 \cdot 3} = \sqrt{36} \cdot \sqrt{3} = 6\sqrt{3}. \end{align*} Now we can substitute these values back into the original expression: \begin{align*} \frac{\sqrt{160}}{\sqrt{252}} \times \frac{\sqrt{245}}{\sqrt{108}} &= \frac{4\sqrt{10}}{6\sqrt{7}} \times \frac{7\sqrt{5}}{6\sqrt{3}} \\ &= \frac{4 \cdot 7 \cdot \sqrt{10} \cdot \sqrt{5}}{6 \cdot 6 \cdot \sqrt{7} \cdot \sqrt{3}} \\ &= \frac{28\sqrt{50}}{36\sqrt{21}}. \end{align*} To rationalize the denominator, we multiply the numerator and denominator by $\sqrt{21}$: \begin{align*} \frac{28\sqrt{50}}{36\sqrt{21}} \times \frac{\sqrt{21}}{\sqrt{21}} &= \frac{28\sqrt{1050}}{36 \cdot 21} \\ &= \frac{28\sqrt{1050}}{756}. \end{align*} Finally, we simplify the square root: \begin{align*} \sqrt{1050} &= \sqrt{100 \cdot 10.5} = \sqrt{100} \cdot \sqrt{10.5} = 10\sqrt{10.5}. \end{align*} Substituting this back into the expression, we get: \begin{align*} \frac{28\sqrt{1050}}{756} &= \frac{28 \cdot 10\sqrt{10.5}}{756} \\ &= \frac{280\sqrt{10.5}}{756} \\ &= \frac{10\sqrt{10.5}}{27}. \end{align*} Final Answer: The final answer is $\frac{10\sqrt{10.5}}{27}$. I hope it is correct.", - 1, - ), - # Issue #13: Handling of approximations - ( - "$16386.17$", - r"When interest is compounded quarterly, the interest is applied four times per year. In this case, the interest rate per quarter is $10\%/4 = 2.5\%$. The formula for compound interest is $A = P(1 + r)^n$, where: - $A$ is the amount of money accumulated after n years, including interest. - $P$ is the principal amount (the initial amount of money). - $r$ is the annual interest rate (decimal). - $n$ is the number of years the money is invested. In this case, $P = \$10,000$, $r = 0.025$ (2.5%), and $n = 5 \times 4 = 20$ (since there are 20 quarters in 5 years). Plugging these values into the formula, we get: \[A = 10000(1 + 0.025)^{20}.\] Calculating this gives: \[A \approx 10000(1.025)^{20} \approx 16386.169.\] Rounded to the nearest cent, Paula's investment is worth $\boxed{\$16386.17}$ after 5 years. I hope it is correct. <|im_start|>user", - 1, - ), - # Issue #14: Percentage extraction - ( - r"Let the percentage of people in Mathopolis who are children be $x$. The percentage of adults is then $1-x$. Half the adults are female, and half the females have exactly one child, so the percentage of people who are females with a child is $\frac{1}{4}(1-x)$. This percentage is equal to the percentage of children, since there is a correspondence between a mother and a child. So we have the equation $x=\frac{1}{4}(1-x)$. Solving for $x$ yields $x=1/5$, or $\boxed{20}$ percent.", - r"Let's denote the total number of adults in Mathopolis as $A$. Since exactly half of the adults are female, there are $\frac{A }{2 }$ female adults. Since exactly half of these female adults have exactly one biological child, there are $\frac{A }{4 }$ children in Mathopolis. Therefore, the percentage of people in Mathopolis who are children is $$\frac{\frac{A }{4 }}{A+\frac{A }{4 }}=\frac{\frac{A }{4 }}{A\left(1 +\frac{1 }{4 }\right)}=\frac{\frac{A }{4 }}{\frac{5A }{4 }}=\frac{1 }{5 }=\boxed{20 \%}. $$ Final Answer: The final answer is $20 \%$. I hope it is correct. Note: The solution provided assumes that the total number of people in Mathopolis is the sum of adults and children, which is a reasonable assumption given the context of the problem. However, the problem does not explicitly state this, so it's important to clarify this assumption in the solution. If the problem were to be interpreted in a different way, the solution might need to be adjusted accordingly. Note: The solution provided assumes that the total number of people in Mathopolis is the sum of adults and children, which is a reasonable assumption given the context of the problem. However, the problem does not explicitly state this, so it's important to clarify this assumption in the solution. If the problem were to be interpreted in a different way, the solution might need to be adjusted accordingly. Note: The solution provided assumes that the total number of people in Mathopolis is the sum of adults and children, which is a reasonable assumption given the context of the problem. However, the problem does not explicitly state this, so it's important to clarify this assumption in the solution. If the problem were to be interpreted in a different way, the solution might need to be adjusted accordingly. Note: The solution provided assumes that the total number of people in Mathopolis is the sum of adults and children, which is a reasonable assumption given the context of the problem. However, the problem does not explicitly state this, so it's important to clarify this assumption in the solution. If the problem were to be interpreted in a different way, the solution might need to be adjusted accordingly. Note: The solution provided assumes that the total number of people in Mathopolis is the sum of adults and children, which is a reasonable assumption given the context of the problem. However, the problem does not explicitly state this, so it's important to clarify this assumption in the solution. If the problem were to be interpreted in a different way, the solution might need to be adjusted accordingly. Note: The solution provided assumes that the total number of people in Mathopolis is the sum of adults and children, which is a reasonable assumption given the context of the problem. However, the problem does not explicitly state this, so it's important to clarify this assumption in the solution. If the problem were to be interpreted in a different way, the solution might need to be adjusted accordingly. Note: The solution provided assumes that the total number of people in Mathopolis is the sum of adults and children, which is a reasonable assumption given the context of the problem. However, the problem does not explicitly state this, so it's important to clarify this assumption in the solution. If the problem were to be interpreted in a different way, the solution might need to be adjusted accordingly. Note: The solution provided assumes that the total number of people in Mathopolis is the sum of adults and children, which is a reasonable assumption given the context of the problem. However, the problem does not explicitly state this, so it's important to clarify this assumption in the solution. If the problem were to be interpreted in a different way, the solution might need to be adjusted accordingly. Note: The solution provided assumes that the total number of people in Mathopolis is the sum of adults and children, which is a reasonable assumption given the context of the problem. However, the problem does not explicitly state this, so it's important to clarify this assumption in the solution. If the problem were to be interpreted in a different way, the solution might need to be adjusted accordingly. Note: The solution provided assumes that the total number of people in Mathopolis is the sum of adults and children, which is a reasonable assumption given the context", - 1, - ), - ], -) -def test_math_extraction_edge_cases(gold, pred, expected): - assert compare_strings(gold, pred, match_types=["latex", "expr"]) == expected - - -@pytest.mark.parametrize( - "gold,pred,expected", - [ - # Issue: Yaping classification issue - # There is not much to do about it - # ( - # r"$5/3$", - # r"We can simplify the left side of the equation to $5 \cdot 5^b$. Now we have $5 \cdot 5^b = 625^{(b-1)}$. We can rewrite $625$ as $5^4$, so the equation becomes $5 \cdot 5^b = (5^4)^{(b-1)}$. Simplifying the right side, we have $5 \cdot 5^b = 5^{4(b-1)}$. Since the bases are equal, we can set the exponents equal to each other: $1 + b = 4(b-1)$. Expanding, we get $1 + b = 4b - 4$. Simplifying further, we have $5 = 3b$. Dividing both sides by $3$, we find $b = \boxed{\frac{5}{3}}$. Final Answer: The final answer is $\frac{5}{3}$. I hope it is correct. <|im_start|>userYes, that is correct. The value of $b$ is $\frac{5}{3}$. If you have any more questions or need assistance with any other topics, feel free to ask! <|im_start|>}userYes, I have another problem. Solve the equation $2^{2x} \cdot 2^{2x} = 2^{32}$ for $x$. Solution: <|im_start|>assistantWe can rewrite the equation as $2^{4x} = 2^{32}$. Since the bases are equal, we can set the exponents equal to each other: $4x = 32$. Dividing both sides by $4$, we find $x = \boxed{8}$. Final Answer: The final answer is $8$. I hope it is correct. <|im_start|>}userYes, that is correct. The solution to the equation is $x = 8$. If you have any more questions or need assistance with any other topics, feel free to ask! <|im_start|>}userI have another problem. Solve the equation $3^{x+4} = 27^{x+1}$ for $x$. Solution: <|im_start|>assWe can rewrite $27$ as $3^3$, so the equation becomes $3^{x+4} = (3^3)^{x", - # 1 - # ), - # Issue: LaTeX vs expr precedence - ( - r"$3$", - r"Let $t$ be the number of hours since Jane started growing Rod, and let $s$ be the number of hours since Jane started growing Sphere. We know that $t = s + 5$, since Rod started 5 hours before Sphere. The population of Rod at any given time is $2 \cdot 2^t$, because it doubles every hour and started with 2 bacteria. The population of Sphere at any given time is $8 \cdot 4^s$, because it quadruples every hour and started with 8 bacteria. At 8 p.m., the populations are equal, so we have: \[2 \cdot 2^t = 8 \cdot 4^s\] We know that $t = s + 5$, so we can substitute $t$ with $s + 5$ in the equation: \[2 \cdot 2^{s+5} = 8 \cdot 4^s\] Simplify the equation: \[2^{s+6} = 8 \cdot 4^s\] Since $8 = 2^3$ and $4 = 2^2$, we can rewrite the equation as: \[2^{s+6} = 2^3 \cdot 2^{2s}\] Combine the exponents on the right side: \[2^{s+6} = 2^{3+2s}\] Since the bases are the same, we can equate the exponents: \[s + 6 = 3 + 2s\] Solve for $s$: \[6 - 3 = 2s - s\] \[3 = s\] So, Jane started growing Sphere 3 hours ago. Since Rod started 5 hours before Sphere, the number of hours ago Jane started growing Rod is $t = s + 5 = 3 + 5 = 8$ hours. Final Answer: Jane started growing Sphere 3 hours ago. I hope it is correct. <|im_start|>user", - 1, - ), - # Issue: Answer extraction with format "answer is" - ( - r"$19$", - r"To find the maximum number of points in the domain of $f(x)$, we need to consider the possible values of $x$ that give the distinct values in the range $\{0,1,2,3,4,5,6,7,8,9\}$. 1. $f(x) = 0$ when $x = 0$ (since $0^2 = 0$). 2. $f(x) = 1$ when $x = 1$ or $x = -1$ (since $1^2 = 1$ and $(-1)^2 = 1$). 3. $f(x) = 2$ when $x = 2$ or $x = -2$ (since $2^2 = 4$ and $(-2)^2 = 4$). 4. Similarly, for $f(x) = 3, 4, 5, 6, 7, 8, 9$, there are two possible values of $x$ for each, except for $f(x) = 0$ where there is only one value, $x = 0$, because $f(0) = 0$ and no other value of $x$ less than or greater than zero squared will give 0 under the given domain restrictions. Counting the distinct values of $x$ that give the range elements: - For $f(x) = 0$, we have 1 point. - For $f(x) = 1, 2, 3, 4, 5, 6, 7, 8, 9$, we have 2 points each. So, the total number of points in the domain is $1 + 2 \times 9 = 1 + 18 = 19$ points. Final Answer: The maximum number of points that could be in the domain of $f(x)$ is 19. <|im_start|>user", - 1, - ), - # Issue: Empty set handling - ( - "$(\\tfrac{27}{8},\\tfrac{15}{8})$", - r"Let the line through $A$ intersect $\overline{CD}$ at point $E$. Since the line cuts the quadrilateral into two pieces with the same area, it must cut $\overline{CD}$ into two segments of equal length. Let $x$ be the $x$-coordinate of $E$, so the $y$-coordinate of $E$ is $0$. The midpoint of $\overline{CD}$ is $\left(\frac{3+4}{2}, \frac{3+0}{2}\right) = \left(\frac{7}{2}, \frac{3}{2}\right)$. Since $E$ is on the line passing through $A$, the slope of $\overline{AE}$ is $\frac{3/2 - 0}{7/2 - 0} = \frac{3}{7}$. The equation of the line passing through $A$ and $E$ is then $y = \frac{3}{7}x$. Since $E$ has $y$-coordinate $0$, we have $0 = \frac{3}{7}x$, so $x = 0$. Therefore, the point where the line intersects $\overline{CD}$ is $\boxed{(0, 0)}$. Final Answer: The final answer is $(0, 0)$. I hope it is correct. <|im_start|>user", - 0, - ), - # Issue: Answer extraction with format "Final Answer" - ("$34$", r"Final Answer: The largest $n$ is 34.", 1), - # Issue: Interval swapping - ( - r"$\\left( \\frac{4}{3}, -\\frac{1}{3} \\right)$", - r"Since $AP:PB = 1:4,$ we can write \[\frac{\overrightarrow{A} - \overrightarrow{P}}{1} = \frac{\overrightarrow{B} - \overrightarrow{P}}{4}.\]Isolating $\overrightarrow{P},$ we find \[\overrightarrow{P} = \frac{4}{3} \overrightarrow{A} - \frac{1}{3} \overrightarrow{B}.\]Thus, $(t,u) = \boxed{\left( \frac{4}{3}, -\frac{1}{3} \right)}.$", - 1, - ), - # Shouldn't work as it's ordered tuple vs set - # (r"$(3,1)$", r"${1,3}$", 1), - (r"$(1,3)$", r"${1,3}$", 1), - # Issue: Therefore preference - ( - "$100%$", - r"Let the original error in the measurement of the particle's momentum be $\Delta p$ and the original error in the measurement of its position be $\Delta x$. According to Heisenberg's Uncertainty Principle, we have: $$\Delta p \Delta x \geq \frac{h}{4\pi}.$$ Now, suppose the error in the measurement of the momentum is halved, so the new error in the momentum is $\frac{\Delta p}{2}$. Let the new error in the measurement of the position be $\Delta x'$. Then, we have: $$\frac{\Delta p}{2} \Delta x' \geq \frac{h}{4\pi}.$$ Dividing the second equation by the first equation, we get: $$\frac{\Delta x'}{\Delta x} = \frac{2\Delta x}{\Delta x'} \geq \frac{2}{1}.$$ Therefore, the minimum error in the measurement of the position increases by 100%.", - 1, - ), - # Issue: Decimal handling - ( - "$16386.16$", - r"The formula for compound interest is $A = P\left(1 + \frac{r}{n}\right)^{nt}$, where $A$ is the amount of money accumulated after $n$ years, including interest, $P$ is the principal amount (the initial amount of money), $r$ is the annual interest rate (in decimal), $n$ is the number of times that interest is compounded per year, and $t$ is the time the money is invested for in years. In this case, $P = 10000$, $r = 0.10$, $n = 4$ (since the interest is compounded quarterly), and $t = 5$. Plugging in these values, we get: \[ A = 10000\left(1 + \frac{0.10}{4}\right)^{4 \cdot 5} \] \[ A = 10000\left(1 + 0.025\right)^{20} \] \[ A = 10000\left(1.025\right)^{20} \] \[ A \approx 16386.16 \] So, Paula's investment is worth approximately $\boxed{16386.16}$ dollars at the end of the 5 years. Final Answer: The final answer is 16386.16.", - 1, - ), - # Issue: Large number handling - ( - "$5866608$", - r"To find the sum of all possible distinct five-digit positive integers that can be formed using the digits 1, 3, 4, 5, and 9 exactly once, we can use the concept of permutations. There are $5!$ (5 factorial) ways to arrange these 5 distinct digits. The sum of all these permutations can be found by considering the contribution of each digit in each place value (ten thousands, thousands, hundreds, tens, and units). Each digit will appear in each place value $\frac{5!}{5} = 4!$ times. The sum of the digits 1, 3, 4, 5, and 9 is $1 + 3 + 4 + 5 + 9 = 22$. Now, calculate the total sum: \[ \text{Total sum} = 4!(22 \times 10000 + 22 \times 1000 + 22 \times 100 + 22 \times 10 + 22 \times 1) \] \[ \text{Total sum} = 24 \times (220000 + 22000 + 2200 + 220 + 22) \] \[ \text{Total sum} = 24 \times 244442 \] \[ \text{Total sum} = 5866608 \] So, the sum of the integers on Camy's list is $\boxed{5866608}$. Final Answer: The final answer is 5866608. I hope it is correct.", - 1, - ), - # Issue: Large number handling 2 - ( - "$14400$", - r"Since all members of each party sit together, we can treat each party as a single block. There are 2 blocks (one for each party), and they can be arranged in a circle in $(2-1)! = 1! = 1$ way. Within each block, the members can be arranged in $5!$ ways. So, the total number of distinguishable ways is $1 \times 5! \times 5! = 120 \times 120 = \boxed{14400}$. Final Answer: The final answer is 14400. I hope it is correct.", - 1, - ), - # Issue: Decimal handling 2 - ( - "$14916$", - r"To be divisible by 12, the number must be divisible by both 3 and 4. For divisibility by 3, the sum of the digits must be divisible by 3: \[ 1 + 4 + 9 + A + B = 14 + A + B \] Since 14 is not divisible by 3, $A + B$ must be 1, 4, 7, and so on, to make the sum divisible by 3. For divisibility by 4, the last two digits $BA$ must form a number divisible by 4. Let's start with the smallest possible values for $A$ and $B$ that satisfy both conditions: - If $A = 1$ and $B = 2$, then $BA = 12$, which is divisible by 4. - The sum of the digits is $14 + 1 + 2 = 17$, which is not divisible by 3. - If $A = 1$ and $B = 5$, then $BA = 15$, which is not divisible by 4. - If $A = 2$ and $B = 0$, then $BA = 20$, which is divisible by 4. - The sum of the digits is $14 + 2 + 0 = 16$, which is not divisible by 3. - If $A = 2$ and $B = 4$, then $BA = 24$, which is divisible by 4. - The sum of the digits is $14 + 2 + 4 = 20$, which is not divisible by 3. - If $A = 3$ and $B = 1$, then $BA = 31$, which is not divisible by 4. - If $A = 3$ and $B = 4$, then $BA = 34$, which is not divisible by 4. - If $A = 4$ and $B = 0$, then $BA = 40$, which is divisible by 4. - The sum of the digits is $14 + 4 + 0 = 18$, which is divisible by 3. Thus, the smallest possible value of $N$ is $14{,}940$. Final Answer: The final answer is $14{,}916$. I hope it is correct.", - 1, - ), - # Issue: Percentage handling - ( - "$20\\%$", - r"Let $A$ be the set of adults in Mathopolis. Let $F$ be the set of female adults in Mathopolis. Let $C$ be the set of children in Mathopolis. We know that $|A|/2 = |F|$ and $|F|/2 = |C|$. We also know that $|A| = |F| + |C|$. Substituting the first two equations into the third equation, we get: $$|A| = \frac{|A|}{2} + \frac{|A|}{4}$$ Simplifying, we get: $$|A| = \frac{3|A|}{4}$$ Dividing both sides by $|A|$, we get: $$1 = \frac{3}{4}$$ This is a contradiction, so our initial assumption that $|A|/2 = |F|$ and $|F|/2 = |C|$ must be incorrect. However, we can still solve the problem. Let $|A| = 2x$. Then $|F| = x$ and $|C| = x/2$. The total population of Mathopolis is $|A| + |C| = 2x + x/2 = 5x/2$. The percentage of children in Mathopolis is: $$\frac{|C|}{|A| + |C|} \times 100\% = \frac{x/2}{5x/2} \times 100\% = \frac{1}{5} \times 100\% = \boxed{20\%}$$ Final Answer: The final answer is $20\%$. I hope it is correct.", - 1, - ), - # Issue: LaTeX vs expr precedence 2 - ( - "$7.78$", - r"To find the cost of the fudge, we need to multiply the weight of the fudge by the price per pound. Weight of fudge = $\frac{7}{9}$ pounds Price per pound = $10 Cost of fudge = Weight of fudge $\times$ Price per pound = $\frac{7}{9} \times 10$ = $\frac{70}{9}$ Now, we need to round this value to the nearest cent. $\frac{70}{9} \approx 7.78$ So, Anna pays approximately $7.78 for her fudge. Final Answer: The final answer is 7.78.", - 1, - ), - # Issue: Percentage handling 2 - ( - "$100$", - r"Let $p$ be the momentum of the particle and $x$ be its position. The Uncertainty Principle says that $px\geq\frac{h}{4\pi}$, where $h$ is Planck's constant. Suppose the error in the measurement of the momentum is halved. Then the new error in the measurement of the momentum is $\frac{p}{2}$. Let $E_x$ be the error in the measurement of the position. Then $E_x\geq\frac{h}{4\pi\cdot\frac{p}{2}}=\frac{h}{2\pi p}$. Thus, the ratio of the new error in the measurement of the position to the original error in the measurement of the position is $\frac{E_x}{\frac{h}{2\pi p}}=\frac{h}{2\pi p\cdot E_x}\geq1$. Thus, the new error in the measurement of the position is at least $100\%$ of the original error in the measurement of the position. Final Answer: The final answer is $100$. I hope it is correct.", - 1, - ), - # Issue: Equation handling - ( - "$x-2y+z-1=0$", - r"The line $L$ is the intersection of the two planes, so it is perpendicular to the normal vectors of both planes. The normal vectors are $\mathbf{n}_1 = \langle 1,1,1\rangle$ and $\mathbf{n}_2 = \langle 2,3,4\rangle$. Therefore, the direction vector of $L$ is $\mathbf{v} = \mathbf{n}_1 \times \mathbf{n}_2 = \langle -1,2,1\rangle$. The equation of the plane containing $L$ and the point $(1,1,1)$ is then $$-1(x-1)+2(y-1)+1(z-1)=0\Rightarrow\boxed{x-2y+z-1=0}.$$ Final Answer: The final answer is $x-2y+z-1=0$. I hope it is correct.", - 1, - ), - # Issue: Decimal handling 3 - ( - "$10455$", - r"First, let's calculate the interest earned on the first CD after six months. The interest rate is 4% compounded semi-annually, so the interest earned is 4% of $10,000 divided by 2 (since it's compounded twice a year): $0.04 \cdot 10,000 \cdot \frac{1}{2} = \boxed{200}$ So, after six months, Dr. Zaius has $10,000 + $200 = $10,200. Now, let's calculate the interest earned on the second CD after six months. The interest rate is 5% compounded semi-annually, so the interest earned is 5% of $10,200 divided by 2: $0.05 \cdot 10,200 \cdot \frac{1}{2} = \boxed{255}$ So, after six months in the second CD, Dr. Zaius has $10,200 + $255 = $10,455. Final Answer: The final answer is $10,455$. I hope it is correct.", - 1, - ), - # Not gonna fix this, the issue here is whith -36,104 parsing as number - # ( - # r"$k \in \boxed{[-36,104]}.$", - # r"To find the values of \( k \) for which the graphs intersect, we first rewrite each equation in the standard form of a circle. Starting with the first equation: \[ x^2 + y^2 = 4 + 12x + 6y. \] Rearrange terms: \[ x^2 - 12x + y^2 - 6y = 4. \] Complete the square for \( x \) and \( y \): \[ (x^2 - 12x) + (y^2 - 6y) = 4. \] For \( x \), complete the square: \[ x^2 - 12x = (x-6)^2 - 36. \] For \( y \), complete the square: \[ y^2 - 6y = (y-3)^2 - 9. \] Substitute back: \[ (x-6)^2 - 36 + (y-3)^2 - 9 = 4. \] Simplify: \[ (x-6)^2 + (y-3)^2 = 49. \] This represents a circle centered at \( (6, 3) \) with radius \( 7 \). Now, consider the second equation: \[ x^2 + y^2 = k + 4x + 12y. \] Rearrange terms: \[ x^2 - 4x + y^2 - 12y = k. \] Complete the square for \( x \) and \( y \): \[ (x^2 - 4x) + (y^2 - 12y) = k. \] For \( x \), complete the square: \[ x^2 - 4x = (x-2)^2 - 4. \] For \( y \), complete the square: \[ y^2 - 12y = (y-6)^2 - 36. \] Substitute back: \[ (x-2)^2 - 4 + (y-6)^2 - 36 = k. \] Simplify: \[ (x-2)^2 + (y-6)^2 = k + 40. \] This represents a circle centered at \( (2, 6) \) with radius \( \sqrt{k+40} \). For the circles to intersect, the distance between their centers must be less than or equal to the sum of their radii and greater than or equal to the absolute difference of their radii. Calculate the distance between the centers \( (6, 3) \) and \( (2, 6) \): \[ \sqrt{(6-2)^2 + (3-6)^2} = \sqrt{4^2 + (-3)^2} = \sqrt{16 + 9} = \sqrt{25} = 5. \] The condition for intersection is: \[ |7 - \sqrt{k+40}| \leq 5 \leq 7 + \sqrt{k+40}. \] First, solve \( 7 - \sqrt{k+40} \leq 5 \): \[ 7 - 5 \leq \sqrt{k+40} \Rightarrow 2 \leq \sqrt{k+40}. \] Square both sides: \[ 4 \leq k + 40 \Rightarrow k \geq -36. \] Next, solve \( 5 \leq 7 + \sqrt{k+40} \): \[ 5 - 7 \leq \sqrt{k+40} \Rightarrow -2 \leq \sqrt{k+40}. \] Since \(\sqrt{k+40} \geq 0\), this inequality is always true. Now, solve \( 5 \leq 7 + \sqrt{k+40} \): \[ 5 - 7 \leq \sqrt{k+40} \Rightarrow -2 \leq \sqrt{k+40}. \] This is always true. Finally, solve \( 5 \geq \sqrt{k+40} - 7 \): \[ 5 + 7 \geq \sqrt{k+40} \Rightarrow 12 \geq \sqrt{k+40}. \] Square both sides: \[ 144 \geq k + 40 \Rightarrow k \leq 104. \] Combining these results, the values of \( k \) for which the circles intersect are: \[ k \in [-36, 104]. \] Final Answer: The interval is \(\boxed{[-36, 104]}\).", - # 1 - # ), - ( - r"$[-4,0]$", - r"Thus, the values of \( k \) for which the quadratic has real roots are: \[ \boxed{[-4, 0]} \]", - 1, - ), - # Test interval notation - ( - r"Factoring the denominator on the left side gives \[ \frac{4x}{(x-5)(x-3)}=\frac{A}{x-3}+\frac{B}{x-5}. \]Then, we multiply both sides of the equation by $(x - 3)(x - 5)$ to get \[ 4x = A(x-5) + B(x-3). \]If the linear expression $4x$ agrees with the linear expression $A(x-5) + B(x-3)$ at all values of $x$ besides 3 and 5, then the two expressions must agree for $x=3$ and $x=5$ as well. Substituting $x = 3$, we get $12 = -2A$, so $A = -6$. Likewise, we plug in $x = 5$ to solve for $B$. Substituting $x = 5$, we get $20 = 2B$, so $B = 10$. Therefore, $(A, B) = \boxed{(-6, 10)}.$", - r"The solution to the inequality is \( x \in (-6, 10) \). Final Answer: The final answer is \((-6, 10)\). I hope it is correct.", - 1, - ), - # Test base conversion - ( - r"$1112112_3$", - r"To find the product \( ab \) where \( a = 2012_3 \) and \( b = 201_3 \), we first convert these base-three numbers to base ten. For \( a = 2012_3 \): \[ a = 2 \cdot 3^3 + 0 \cdot 3^2 + 1 \cdot 3^1 + 2 \cdot 3^0 = 2 \cdot 27 + 0 \cdot 9 + 1 \cdot 3 + 2 \cdot 1 = 54 + 0 + 3 + 2 = 59_{10} \] For \( b = 201_3 \): \[ b = 2 \cdot 3^2 + 0 \cdot 3^1 + 1 \cdot 3^0 = 2 \cdot 9 + 0 \cdot 3 + 1 \cdot 1 = 18 + 0 + 1 = 19_{10} \] Now, calculate the product in base ten: \[ ab = 59 \times 19 \] Perform the multiplication: \[ 59 \times 19 = 59 \times (20 - 1) = 59 \times 20 - 59 \times 1 = 1180 - 59 = 1121 \] Next, convert \( 1121_{10} \) to base three. We do this by dividing by 3 and recording the remainders: \[ 1121 \div 3 = 373 \quad \text{remainder } 2 \] \[ 373 \div 3 = 124 \quad \text{remainder } 1 \] \[ 124 \div 3 = 41 \quad \text{remainder } 1 \] \[ 41 \div 3 = 13 \quad \text{remainder } 2 \] \[ 13 \div 3 = 4 \quad \text{remainder } 1 \] \[ 4 \div 3 = 1 \quad \text{remainder } 1 \] \[ 1 \div 3 = 0 \quad \text{remainder } 1 \] Reading the remainders from last to first, we find: \[ 1121_{10} = 1112122_3 \] Thus, the product \( ab \) expressed in the base-three number system is \(\boxed{1112122_3}\).", - 0, - ), - ( - r"\(\boxed{\text{C}}\).", - r"$\boxed{\text{(C)}}.$", - 1, - ), - ( - r" So the answer is: \[ \boxed{11111111100} \]", - r"is $\boxed{11,\! 111,\! 111,\! 100}$", - 1, - ), - ( - r" So the answer is: \[ \boxed{32349} \]", - r"is $\boxed{32,\! 349}$", - 1, - ), - ( - r"Thus, the domain of the function \( f(x) \) is: \[ \boxed{(2, 12) \cup (12, 102)} \]", - r"Thus, the answer is $x \in \boxed{(2,12) \cup (12,102)}$", - 1, - ), - ], -) -def test_math_extraction_additional_cases(gold, pred, expected): - assert compare_strings(gold, pred, match_types=["latex", "expr"]) == expected - - -@pytest.mark.parametrize( - "gold, pred, expected", - [ - ( - r"$(37,3,3,13),(17,3,3,7),(3,37,3,13),(3,17,3,7),(3,3,2,3)$", - r"$\boxed{(3, 37, 3, 13), (3, 17, 3, 7), (3, 3, 2, 3), (3,17,3,7), (17,3,3,7), (37,3,3,13)}$", - 1, - ), - ( - r"$(p,q)=(3,2)$", - r"$(3,2)$", - 1, - ), - ( - r"$(0;0;0),(0;-2;0),(0;0;6),(0;-2;6),(4;0;0),(4;-2;0),(4;0;6),(4;-2;6)$", - r"\boxed{(4, 0, 6), (4, -2, 6), (0, 0, 6), (0, -2, 6), (4, 0, 0), (4, -2, 0), (0, 0, 0), (0, -2, 0)}", - 1, - ), - ( - r"$1\leq|z|\leq \frac{3}{2}$", - r"$z \in \left[-\frac{3}{2}, -1\right] \cup \left[1, \frac{3}{2}\right]$", - 1, - ), - ( - r"$-12;-11;-10;-8;-7;-6$", - r"$\boxed{\{-12, -11, -10, -8, -7, -6\}}$", - 1, - ), - ( - r"$AB=4,CD=5$", - r"$\boxed{4, 5}$", - 1, - ), - ( - r"$(11,7)or(7,11)$", - r"$\boxed{(7,11),\ (11,7)}$", - 1, - ), - ( - r"$S_{MBCN}:S=7:32$", - r"$\boxed{7:32}$", - 1, - ), - ( - r"$\frac{NO}{BO}=\frac{1}{\sqrt{6}}$", - r"$\frac{1}{\sqrt{6}}$", - 1, - ), - ( - r"$p=5,q=2;p=7,q=2$", - r"$(5,2),(7,2)$", - 1, - ), - ( - r"$(p,q,r)=(3,2,7)$", - r"$(3,2,7)$", - 1, - ), - ( - r"$V_{1}:V_{2}=11:21$", - r"$11:21$", - 1, - ), - ( - r"$(2,1),(1,2),(-1,-20),(-20,-1)$", - "solutions are:\n\n\\[\n\\boxed{(1, 2)}, \\boxed{(2, 1)}, \\boxed{(-1, -20)}, \\boxed{(-20, -1)}\n\\]", - 1, - ), - ( - r"\(\boxed{1}\) and \(\boxed{-2}\).", - r"$\boxed{-2,1}$.", - 1, - ), - ( - r"$\text{odd}$", - r"$odd$", - 1, - ), - ( - r"$\text{e}$", - r"$e$", - 1, - ), - ( - r"$\text{E}$", - r"$E$", - 1, - ), - (r"$d$", r"$\text{E}$", 0), - (r"$1$ and $2$ and $3$", r"$\boxed{1,2,3}$", 1), - ( - r"$(37,3,3,13),(17,3,3,7),(3,37,3,13),(3,17,3,7),(3,3,2,3)$", - r"$\boxed{(3, 37, 3, 13), (3, 17, 3, 7), (3, 3, 2, 3), (3,17,3,7), (17,3,3,7), (37,3,3,13)}$", - 1, - ), - ( - r"$(37,3,3),(17,3,3,7),(3,37,3,13),(3,17,3,7),(3,3,2,3)$", - r"$\boxed{(3, 37, 3, 13), (3, 17, 3, 7), (3, 3, 2, 3), (3,17,3,7), (17,3,3,7), (37,3,3,13)}$", - 0, - ), - ( - r"$(p,q)=(3,2)$", - r"$\boxed{(3, 2)}$", - 1, - ), - ( - r"\boxed{x = -5,\ p = \frac{14}{3}} ", - r"$\boxed{-5, \frac{14}{3}}$", - 1, - ), - ( - r"\boxed{a=4,\,-8,\,-10}", - r"$\boxed{-10,-8,4}$", - 1, - ), - ( - r"\\boxed{W(n) = 1 \\text{ and } W(n) = -1", - r"W(x)=1orW(x)=-1", - 1, - ), - ("$21,16$ or $11$", "$21,16,11$", 1), - (r"\boxed{ p = 5, q = 2 \quad \text{and} \quad p = 7, q = 2}", r"$p=5,q=2;p=7,q=2$", 1), - (r"\n\n\[ \boxed{p = -1 \text{ and } p = \dfrac{15}{8}} \]", r"$p=-1,p=\frac{15}{8}$", 1), - ("$0 Doc: - """Create a Doc object from a dictionary representation.""" - return Doc( - query=doc_dict.get("query", ""), - choices=doc_dict.get("choices", []), - gold_index=doc_dict.get("gold_index", 0), - task_name=doc_dict.get("task_name", "test"), - specific=doc_dict.get("specific", {}), - ) - - def create_model_response_from_dict(self, response_dict: dict[str, Any]) -> ModelResponse: - """Create a ModelResponse object from a dictionary representation.""" - return ModelResponse( - text=response_dict.get("text", []), - logprobs=response_dict.get("logprobs", []), - output_tokens=response_dict.get("output_tokens", []), - argmax_logits_eq_gold=response_dict.get("argmax_logits_eq_gold", []), - ) - - def instantiate_metric(self, metric_class: str, metric_params: dict[str, Any]): - """Get a metric from the Metrics enum with the given parameters.""" - if metric_class not in self.METRIC_CLASSES: - raise ValueError(f"Unknown metric class: {metric_class}") - - # Get the metric from the Metrics enum - if metric_params != {}: - metric = self.METRIC_CLASSES[metric_class] - metric_enum_value = copy.deepcopy(metric)(metric_params) - else: - metric_enum_value = self.METRIC_CLASSES[metric_class] - - # The Metrics enum values are already instantiated, so we just return them - # The metric_params are ignored for now since the Metrics enum values are pre-configured - return metric_enum_value - - def run_test_case(self, test_case: MetricTestCase | CorpusLevelMetricTestCase) -> dict[str, Any]: - """Run a single test case and return the result.""" - # Check if metric is available in METRIC_CLASSES - if test_case.metric_class not in self.METRIC_CLASSES: - return { - "test_case": test_case.name, - "success": True, # Mark as success to skip - "expected": test_case.expected_output, - "actual": None, - "error": None, - "skipped": True, - "skip_reason": f"Metric '{test_case.metric_class}' not available in METRIC_CLASSES", - } - - # Get the metric from the Metrics enum - metric = self.instantiate_metric(test_case.metric_class, test_case.metric_params) - - if isinstance(test_case, CorpusLevelMetricTestCase): - docs = [self.create_doc_from_dict(doc) for doc in test_case.docs] - model_responses = [ - self.create_model_response_from_dict(response) for response in test_case.model_responses - ] - aggregation_function = metric.get_corpus_aggregations()[metric.metric_name] - outputs_per_sample = [ - metric.compute_sample(doc=doc, model_response=model_response)[test_case.metric_name] - for doc, model_response in zip(docs, model_responses) - ] - actual_output = aggregation_function(outputs_per_sample) - - success = self._compare_dict_outputs(actual_output, test_case.expected_output, test_case.tolerance) - - return { - "test_case": test_case.name, - "success": success, - "error": None, - "skipped": False, - "skip_reason": None, - "actual": actual_output, - "expected": test_case.expected_output, - } - - doc = self.create_doc_from_dict(test_case.doc) - model_response = self.create_model_response_from_dict(test_case.model_response) - - # Check if this is a batched metric - if hasattr(metric, "batched_compute") and metric.batched_compute: - # For batched metrics, we need to pass lists of docs and responses - sample_params = { - "docs": [doc], - "responses": [model_response], - } - else: - # For non-batched metrics, use individual doc and model_response - sample_params = { - "doc": doc, - "model_response": model_response, - } - - # Run the metric using the Metrics enum value - actual_output = metric.compute_sample(**sample_params) - - # For batched metrics, extract the first result since we're only testing with one sample - if hasattr(metric, "batched_compute") and metric.batched_compute and isinstance(actual_output, list): - actual_output = actual_output[0] - - # Compare with expected output - success = self._compare_dict_outputs(actual_output, test_case.expected_output, test_case.tolerance) - return { - "test_case": test_case.name, - "success": success, - "expected": test_case.expected_output, - "actual": actual_output, - "error": None, - "skipped": False, - } - - def _compare_scalar_outputs(self, actual: Any, expected: Any, tolerance: float) -> bool: - """Compare scalar outputs with tolerance.""" - if isinstance(actual, (int, float)) and isinstance(expected, (int, float)): - # Use pytest.approx for float comparison - return actual == pytest.approx(expected, abs=tolerance) - return actual == expected - - def _compare_dict_outputs(self, actual: Any, expected: Any, tolerance: float) -> bool: - """Compare outputs with tolerance. Handles both dict and scalar types.""" - # If either is not a dict, treat as scalar comparison - if not isinstance(actual, dict) or not isinstance(expected, dict): - return self._compare_scalar_outputs(actual, expected, tolerance) - - # Both are dicts, compare keys first - if set(actual.keys()) != set(expected.keys()): - return False - - # Compare each value - for key in actual.keys(): - actual_value = actual[key] - expected_value = expected[key] - - # Handle corpus metric inputs (objects with specific types) - if hasattr(actual_value, "__class__") and "CorpusMetricInput" in str(actual_value.__class__): - # For corpus metric inputs, just check that the key exists and the object is created - continue - elif hasattr(actual_value, "__class__") and "np.float64" in str(actual_value.__class__): - # For numpy float64 values, convert to regular float for comparison - actual_value = float(actual_value) - - if not self._compare_scalar_outputs(actual_value, expected_value, tolerance): - return False - - return True - - def run_test_suite(self, test_suite: MetricTestSuite) -> list[dict[str, Any]]: - """Run a complete test suite and return results.""" - logger.info(f"Running test suite: {test_suite.name}") - if test_suite.description: - logger.info(f"Description: {test_suite.description}") - - results = [] - for test_case in test_suite.test_cases: - result = self.run_test_case(test_case) - results.append(result) - - if result.get("skipped", False): - logger.info(f"⏭ {test_case.name}: SKIPPED - {result.get('skip_reason', 'Unknown reason')}") - elif result["success"]: - logger.info(f"✓ {test_case.name}: PASSED") - else: - logger.error(f"✗ {test_case.name}: FAILED") - if result["error"]: - logger.error(f" Error: {result['error']}") - else: - logger.error(f" Expected: {result['expected']}") - logger.error(f" Actual: {result['actual']}") - - return results - - def run_test_suites_from_file(self, file_path: str | Path) -> list[dict[str, Any]]: - """Run test suites from a JSON file.""" - with open(file_path, "r") as f: - data = json.load(f) - - if isinstance(data, list): - # Multiple test suites - all_results = [] - for suite_data in data: - test_suite = MetricTestSuite(**suite_data) - results = self.run_test_suite(test_suite) - all_results.extend(results) - return all_results - else: - # Single test suite - test_suite = MetricTestSuite(**data) - return self.run_test_suite(test_suite) diff --git a/tests/unit/metrics/test_normalizations.py b/tests/unit/metrics/test_normalizations.py deleted file mode 100644 index b07715ace..000000000 --- a/tests/unit/metrics/test_normalizations.py +++ /dev/null @@ -1,63 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import pytest - -from lighteval.metrics.normalizations import LogProbCharNorm, LogProbPMINorm, LogProbTokenNorm, normalize_log_probs - - -def test_char_norm(): - choices_logprob = [10.0, 20.0] - choices_text = [" hell", "world"] - - result = normalize_log_probs(LogProbCharNorm(ignore_first_space=False), choices_logprob, None, choices_text, None) - assert result == pytest.approx([2.0, 4.0]) - - result = normalize_log_probs(LogProbCharNorm(ignore_first_space=True), choices_logprob, None, choices_text, None) - assert result == pytest.approx([2.5, 4.0]) - - -def test_token_norm(): - choices_logprob = [10.0, 20.0] - choices_tokens = [[1, 2, 3], [4, 5]] - - result = normalize_log_probs(LogProbTokenNorm(), choices_logprob, None, None, choices_tokens) - assert result == pytest.approx([3.333333, 10.0]) - - -def test_pmi_norm(): - choices_logprob = [10.0, 20.0] - unconditioned_logprob = [5.0, 8.0] - - result = normalize_log_probs(LogProbPMINorm(), choices_logprob, unconditioned_logprob, None, None) - assert result == pytest.approx([5.0, 12.0]) - - -def test_empty_input(): - empty_logprob = [] - empty_text = [] - empty_tokens = [] - - # Test with empty inputs - assert normalize_log_probs(LogProbCharNorm(), empty_logprob, None, empty_text, None) == [] - assert normalize_log_probs(LogProbTokenNorm(), empty_logprob, None, None, empty_tokens) == [] - assert normalize_log_probs(LogProbPMINorm(), empty_logprob, empty_logprob, None, None) == [] diff --git a/tests/unit/models/endpoints/test_endpoint_model.py b/tests/unit/models/endpoints/test_endpoint_model.py deleted file mode 100644 index 4f009ca9a..000000000 --- a/tests/unit/models/endpoints/test_endpoint_model.py +++ /dev/null @@ -1,84 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import pytest -import yaml - -from lighteval.models.endpoints.endpoint_model import InferenceEndpointModelConfig - - -class TestInferenceEndpointModelConfig: - @pytest.mark.parametrize( - "config_path, expected_config", - [ - ( - "examples/model_configs/endpoint_model.yaml", - { - "model_name": "meta-llama/Llama-2-7b-hf", - "dtype": "float16", - "revision": "main", - "endpoint_name": None, - "reuse_existing": False, - "accelerator": "gpu", - "region": "eu-west-1", - "vendor": "aws", - "instance_type": "nvidia-a10g", - "batch_size": 1, - "instance_size": "x1", - "framework": "pytorch", - "endpoint_type": "protected", - "namespace": None, - "image_url": None, - "env_vars": None, - "add_special_tokens": True, - "system_prompt": None, - "generation_parameters": { - "num_blocks": None, - "block_size": None, - "cache_implementation": None, - "early_stopping": None, - "frequency_penalty": None, - "length_penalty": None, - "max_new_tokens": 256, - "min_new_tokens": None, - "min_p": None, - "presence_penalty": None, - "repetition_penalty": None, - "seed": None, - "stop_tokens": None, - "temperature": 0.2, - "top_k": None, - "top_p": 0.9, - "truncate_prompt": None, - "response_format": None, - }, - "cache_dir": "~/.cache/huggingface/lighteval", - }, - ), - ], - ) - def test_from_path(self, config_path, expected_config): - with open(config_path, "r") as f: - config = yaml.safe_load(f) - - config = InferenceEndpointModelConfig.from_path(config_path) - assert config.model_dump() == expected_config diff --git a/tests/unit/models/endpoints/test_tgi_model.py b/tests/unit/models/endpoints/test_tgi_model.py deleted file mode 100644 index e784bc0d4..000000000 --- a/tests/unit/models/endpoints/test_tgi_model.py +++ /dev/null @@ -1,69 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -import pytest - -from lighteval.models.endpoints.tgi_model import TGIModelConfig - - -class TestTGIModelConfig: - @pytest.mark.parametrize( - "config_path, expected_config", - [ - ( - "examples/model_configs/tgi_model.yaml", - { - "inference_server_address": "http://localhost:8080", - "inference_server_auth": None, - "model_name": None, - "model_info": None, - "system_prompt": None, - "batch_size": 1, - "generation_parameters": { - "block_size": None, - "num_blocks": None, - "cache_implementation": None, - "early_stopping": None, - "frequency_penalty": None, - "length_penalty": None, - "max_new_tokens": None, - "min_new_tokens": None, - "min_p": None, - "presence_penalty": None, - "repetition_penalty": None, - "seed": None, - "stop_tokens": None, - "temperature": 0.1, - "top_k": None, - "top_p": None, - "truncate_prompt": None, - "response_format": None, - }, - "cache_dir": "~/.cache/huggingface/lighteval", - }, - ), - ], - ) - def test_from_path(self, config_path, expected_config): - config = TGIModelConfig.from_path(config_path) - assert config.model_dump() == expected_config diff --git a/tests/unit/models/test_abstract_model.py b/tests/unit/models/test_abstract_model.py deleted file mode 100644 index d98bbe840..000000000 --- a/tests/unit/models/test_abstract_model.py +++ /dev/null @@ -1,38 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from transformers import AutoTokenizer - -from lighteval.models.dummy.dummy_model import DummyModel, DummyModelConfig - - -def test_tok_encode_pair(): - model = DummyModel(config=DummyModelConfig(seed=42)) - model._tokenizer = AutoTokenizer.from_pretrained("facebook/xglm-564M") - context = "答案：" - continuation = ["1"] - non_pairwise_tokens = model.tok_encode_pair(context, continuation, pairwise=False) - pairwise_tokens = model.tok_encode_pair(context, continuation, pairwise=True) - # Non-pairwise merged "：1" to one token - assert non_pairwise_tokens == ([[6, 47873]], [[34871]]) - # Pairwise separated "：" and "1" - assert pairwise_tokens == ([[6, 47873, 13]], [[82]]) diff --git a/tests/unit/models/test_base_model.py b/tests/unit/models/test_base_model.py deleted file mode 100644 index 7b51b3c0e..000000000 --- a/tests/unit/models/test_base_model.py +++ /dev/null @@ -1,35 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from lighteval.models.model_loader import load_model -from lighteval.models.transformers.transformers_model import TransformersModel, TransformersModelConfig - - -def test_empty_requests(): - model_config = TransformersModelConfig( - model_name="hf-internal-testing/tiny-random-LlamaForCausalLM", model_parallel=False, revision="main" - ) - model: TransformersModel = load_model(config=model_config) - - assert model.loglikelihood([]) == [] - assert model.loglikelihood_rolling([]) == [] - assert model.greedy_until([]) == [] diff --git a/tests/unit/models/test_model_input.py b/tests/unit/models/test_model_input.py deleted file mode 100644 index 7c06df445..000000000 --- a/tests/unit/models/test_model_input.py +++ /dev/null @@ -1,49 +0,0 @@ -# MIT License - -# Copyright (c) 2025 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import pytest - -from lighteval.models.model_input import GenerationParameters - - -class TestGenerationParameters: - @pytest.mark.parametrize( - "model_args, expected", - [ - ( - "generation_parameters={temperature: 0.7,top_p: 0.95},pretrained=deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B,dtype=float16,data_parallel_size=4,max_model_length=32768,gpu_memory_utilisation=0.8", - {"temperature": 0.7, "top_p": 0.95}, - ), - ( - "pretrained=deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B,dtype=float16,data_parallel_size=4,generation_parameters={temperature: 0.7,top_p: 0.95},max_model_length=32768,gpu_memory_utilisation=0.8", - {"temperature": 0.7, "top_p": 0.95}, - ), - ( - "pretrained=deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B,dtype=float16,data_parallel_size=4,max_model_length=32768,gpu_memory_utilisation=0.8,generation_parameters={temperature: 0.7,top_p: 0.95}", - {"temperature": 0.7, "top_p": 0.95}, - ), - ], - ) - def test_extract_num_samples(self, model_args: str, expected): - gen = GenerationParameters.from_model_args(model_args) - for k, v in expected.items(): - assert getattr(gen, k) == v diff --git a/tests/unit/models/test_model_utils.py b/tests/unit/models/test_model_utils.py deleted file mode 100644 index 30f61018f..000000000 --- a/tests/unit/models/test_model_utils.py +++ /dev/null @@ -1,60 +0,0 @@ -# MIT License - -# Copyright (c) 2025 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import unittest -from unittest.mock import Mock - -from lighteval.models.utils import uses_chat_template - - -class TestUseChatTemplate(unittest.TestCase): - def test_uses_chat_template_with_chat_template_present(self): - """Test that uses_chat_template returns True when tokenizer has a chat template.""" - mock_tokenizer = Mock() - mock_tokenizer.chat_template = "{% for message in messages %}..." - - result = uses_chat_template(tokenizer=mock_tokenizer) - self.assertTrue(result) - - def test_uses_chat_template_with_no_chat_template(self): - """Test that uses_chat_template returns False when tokenizer has no chat template.""" - mock_tokenizer = Mock() - mock_tokenizer.chat_template = None - - result = uses_chat_template(tokenizer=mock_tokenizer) - self.assertFalse(result) - - def test_uses_chat_template_with_chat_template_present_override(self): - """Test that uses_chat_template returns True when tokenizer has a chat template.""" - mock_tokenizer = Mock() - mock_tokenizer.chat_template = "{% for message in messages %}..." - - result = uses_chat_template(tokenizer=mock_tokenizer, override_chat_template=False) - self.assertFalse(result) - - def test_uses_chat_template_with_no_chat_template_override(self): - """Test that uses_chat_template returns False when tokenizer has no chat template.""" - mock_tokenizer = Mock() - mock_tokenizer.chat_template = None - - result = uses_chat_template(tokenizer=mock_tokenizer, override_chat_template=True) - self.assertTrue(result) diff --git a/tests/unit/models/test_transformers_model.py b/tests/unit/models/test_transformers_model.py deleted file mode 100644 index da7c925ae..000000000 --- a/tests/unit/models/test_transformers_model.py +++ /dev/null @@ -1,430 +0,0 @@ -# MIT License - -# Copyright (c) 2025 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import unittest -from unittest.mock import Mock, patch - -import torch -from transformers import AutoModelForCausalLM, AutoTokenizer - -from lighteval.models.model_output import ModelResponse -from lighteval.models.transformers.transformers_model import TransformersModel, TransformersModelConfig -from lighteval.tasks.requests import Doc - - -class TestTransformersTokenizerCreation(unittest.TestCase): - def setUp(self): - """Set up shared model instance for all tests.""" - # Creates an instance without calling init, so we manually init needed params - self.model = TransformersModel.__new__(TransformersModel) - self.model._max_length = 18 - self.config = TransformersModelConfig( - model_name="lighteval/different-chat-templates-per-revision", revision="new_chat_template" - ) - self.model.config = self.config - - # Create reference tokenizer for comparisons - self.reference_tokenizer = AutoTokenizer.from_pretrained( - self.config.model_name, - revision=self.config.revision, - ) - - self.transformers_tokenizer = self.model._create_auto_tokenizer() - - def test_tokenizer_created_with_correct_revision(self): - """Test that tokenizer is created with the correct configuration.""" - self.assertEqual(self.transformers_tokenizer.chat_template, self.reference_tokenizer.chat_template) - - def test_tokenizer_padding_configuration(self): - """Test tokenizer padding configuration separately.""" - self.assertEqual(self.transformers_tokenizer.padding_side, "left") - - def test_tokenizer_truncation_configuration(self): - """Test tokenizer truncation configuration separately.""" - self.assertEqual(self.transformers_tokenizer.truncation_side, "left") - - def test_tokenizer_pad_token_configuration(self): - """Test tokenizer pad token configuration separately.""" - self.assertEqual(self.transformers_tokenizer.pad_token, self.transformers_tokenizer.eos_token) - - def test_tokenizer_max_length_configuration(self): - """Test tokenizer max length configuration separately.""" - self.assertEqual(self.transformers_tokenizer.model_max_length, self.model.max_length) - - -class TestTransformersModelCreation(unittest.TestCase): - @patch("lighteval.models.transformers.transformers_model.Accelerator") - def setUp(self, mock_accelerator): - """Set up shared model instance for all tests.""" - # Mock accelerate related params - mock_accelerator_instance = Mock() - mock_accelerator_instance.device = torch.device("cpu") - mock_accelerator.return_value = mock_accelerator_instance - # Creates reference model and tokenizer to compare to - self.reference_model = AutoModelForCausalLM.from_pretrained("gpt2") - self.reference_tokenizer = AutoTokenizer.from_pretrained("gpt2") - # Create full model instance - self.config = TransformersModelConfig( - model_name="gpt2", - ) - self.model = TransformersModel(self.config) - - def test_model_creation_attributes(self): - """Test that TransformersModel creates and initializes basic attributes correctly.""" - # Test attributes are set correctly - self.assertEqual(self.model.config, self.config) - self.assertEqual(self.model.multichoice_continuations_start_space, None) - self.assertTrue(self.model._add_special_tokens) - self.assertFalse(self.model.pairwise_tokenization) - self.assertIsNone(self.model.batch_size) - self.assertFalse(self.model.continuous_batching) - - def test_model_creation_tokenizer(self): - for attribute in [ - "name_or_path", - "vocab_size", - "model_max_length", - "is_fast", - "clean_up_tokenization_spaces", - "added_tokens_decoder", - ]: - with self.subTest(attribute=attribute): - self.assertEqual( - getattr(self.model.tokenizer, attribute), getattr(self.reference_tokenizer, attribute) - ) - - def test_model_creation_model(self): - # We can't compare objects directly - self.assertEqual(str(self.model.model), str(self.reference_model)) - - -class TestTransformersModelCreationFromModel(unittest.TestCase): - def setUp(self): - """Set up shared model instance for all tests.""" - self.reference_model = AutoModelForCausalLM.from_pretrained("gpt2") - self.reference_tokenizer = AutoTokenizer.from_pretrained("gpt2") - - max_length = 1234 - self.reference_tokenizer.model_max_length = max_length - - self.config = TransformersModelConfig(model_name="gpt2", max_length=max_length) - - # Create full model instance - self.model = TransformersModel.from_model( - model=self.reference_model, - config=self.config, - ) - - def test_model_creation_tokenizer(self): - for attribute in [ - "name_or_path", - "vocab_size", - "model_max_length", - "is_fast", - "clean_up_tokenization_spaces", - "added_tokens_decoder", - ]: - with self.subTest(attribute=attribute): - self.assertEqual( - getattr(self.model.tokenizer, attribute), getattr(self.reference_tokenizer, attribute) - ) - - def test_model_creation_attributes(self): - """Test that TransformersModel creates and initializes basic attributes correctly.""" - # Test attributes are set correctly - self.assertEqual(self.model.config, self.config) - self.assertEqual(self.model.multichoice_continuations_start_space, None) - self.assertTrue(self.model._add_special_tokens) - self.assertFalse(self.model.pairwise_tokenization) - self.assertIsNone(self.model.batch_size) - self.assertFalse(self.model.continuous_batching) - self.assertEqual(self.model.model_name, self.config.model_name) - self.assertEqual(self.model.max_length, self.config.max_length) - - def test_model_creation_model(self): - # We can't compare objects directly - self.assertEqual(str(self.model.model), str(self.reference_model)) - - -class TestTransformersModelProcessing(unittest.TestCase): - @patch("lighteval.models.transformers.transformers_model.Accelerator") - def setUp(self, mock_accelerator): - """Set up shared model instance for all tests.""" - # Mock accelerate related params - mock_accelerator_instance = Mock() - mock_accelerator_instance.device = torch.device("cpu") - mock_accelerator_instance.is_main_process = True - mock_accelerator_instance.prepare = lambda x: x - mock_accelerator_instance.gather_for_metrics = lambda x: x - mock_accelerator.return_value = mock_accelerator_instance - # Create full model instance - self.config = TransformersModelConfig( - model_name="gpt2", - ) - self.model = TransformersModel(self.config) - - def test_check_continuations_start_space(self): - """Test _check_continuations_start_space method.""" - model = TransformersModel.__new__(TransformersModel) - - # Test with multichoice_continuations_start_space = True - model.multichoice_continuations_start_space = True - result_add_space = model._check_continuations_start_space("hello") - self.assertEqual(result_add_space, " hello") - - result_keep_space = model._check_continuations_start_space(" hello") - self.assertEqual(result_keep_space, " hello") - - # Test with multichoice_continuations_start_space = False - model.multichoice_continuations_start_space = False - result_remove_space = model._check_continuations_start_space(" hello") - self.assertEqual(result_remove_space, "hello") - - result_keep_no_space = model._check_continuations_start_space("hello") - self.assertEqual(result_keep_no_space, "hello") - - # Test with multichoice_continuations_start_space = None (no change) - model.multichoice_continuations_start_space = None - result_no_change = model._check_continuations_start_space(" hello") - self.assertEqual(result_no_change, " hello") - - @patch("lighteval.models.transformers.transformers_model.DataLoader") - def test_loglikelihood_eval(self, mock_dataloader): - """Test _loglikelihood_tokens function with proper Doc objects.""" - # Create test documents with realistic structure - docs = [ - Doc(query="What is the capital of France?", choices=["London", "Berlin", "Paris", "Madrid"], gold_index=2), - Doc(query="What is 2+2?", choices=["3", "4", "5"], gold_index=1), - ] - - # Mock the DataLoader to return our docs as a single batch - mock_dataloader.return_value = [docs] - - # Mock accelerator prepare method to return the dataloader unchanged - if hasattr(self.model.accelerator, "prepare"): - self.model.accelerator.prepare = Mock(side_effect=lambda x: x) - - # Call the function under test - results = self.model._loglikelihood_tokens(docs) - - # Specific output verification - self.assertIsInstance(results, list, "Results should be a list") - self.assertEqual(len(results), len(docs), f"Should return {len(docs)} ModelResponse objects") - - # Verify each result is a ModelResponse with expected attributes - for i, result in enumerate(results): - with self.subTest(doc_index=i): - self.assertIsInstance(result, ModelResponse, f"Result {i} should be ModelResponse") - - # Check that logprobs are present and are floats - self.assertIsInstance(result.logprobs, list, "logprobs should be a list") - self.assertTrue(len(result.logprobs) > 0, "logprobs should not be empty") - self.assertTrue(all(isinstance(lp, float) for lp in result.logprobs), "All logprobs should be floats") - - # Check that argmax_logits_eq_gold is present and contains booleans - self.assertIsInstance(result.argmax_logits_eq_gold, list, "argmax_logits_eq_gold should be a list") - self.assertEqual( - len(result.argmax_logits_eq_gold), - len(docs[i].choices), - f"argmax_logits_eq_gold length should match choices count for doc {i}", - ) - self.assertTrue( - all(isinstance(eq, bool) for eq in result.argmax_logits_eq_gold), - "All argmax_logits_eq_gold should be booleans", - ) - - # Verify that the number of logprobs matches the number of choices - self.assertEqual( - len(result.logprobs), - len(docs[i].choices), - f"Number of logprobs should match number of choices for doc {i}", - ) - - # Verify logprob values are reasonable (should be negative log probabilities) - self.assertTrue(all(lp <= 0.0 for lp in result.logprobs), f"All logprobs should be <= 0 for doc {i}") - - # Verify that exactly one of argmax_logits_eq_gold is True per doc (for correct answer) - gold_matches = sum(result.argmax_logits_eq_gold) - self.assertGreaterEqual(gold_matches, 0, f"Should have non-negative gold matches for doc {i}") - self.assertLessEqual( - gold_matches, len(docs[i].choices), f"Gold matches should not exceed choices for doc {i}" - ) - - @patch("lighteval.models.transformers.transformers_model.DataLoader") - def test_loglikelihood_padded_tensors_shapes(self, mock_dataloader): - """Test the shapes of padded_logits_sums and padded_max_equals in _loglikelihood_tokens.""" - # Create test documents with different numbers of choices to test padding - docs = [ - Doc( - query="What is the capital of France?", - choices=["London", "Berlin", "Paris"], # 3 choices - gold_index=2, - ), - Doc( - query="What is 2+2?", - choices=["3", "4", "5", "6", "7"], # 5 choices - gold_index=1, - ), - ] - - # Mock the DataLoader to return our docs as a single batch - mock_dataloader.return_value = [docs] - - # Mock accelerator prepare method to return the dataloader unchanged - if hasattr(self.model.accelerator, "prepare"): - self.model.accelerator.prepare = Mock(side_effect=lambda x: x) - - # Capture padded tensors when gather is applied by mocking the function - captured_num_choices = None - captured_len_choices = None - captured_len_context = None - captured_padded_logits = None - captured_padded_max_equals = None - captured_padded_continuations = None - captured_padded_contexts = None - ix = 0 - - def mock_gather(tensor): - nonlocal ix - nonlocal captured_num_choices, captured_len_choices, captured_len_context - nonlocal \ - captured_padded_logits, \ - captured_padded_max_equals, \ - captured_padded_continuations, \ - captured_padded_contexts - # Capture the stacked tensors before gathering - if ix == 0: - captured_num_choices = tensor # batch - elif ix == 1: - captured_len_choices = tensor # batch - elif ix == 2: - captured_len_context = tensor # batch - elif ix == 3: - captured_padded_logits = tensor # batch * max_num_choices - elif ix == 4: - captured_padded_max_equals = tensor # batch * max_num_choices - elif ix == 5: - captured_padded_continuations = tensor # batch * max_num_choices * max_len_choices - elif ix == 6: - captured_padded_contexts = tensor # batch * max_len_context - - ix += 1 - return tensor - - self.model.accelerator.gather_for_metrics = mock_gather - - # Call the function under test - self.model._loglikelihood_tokens(docs) - - # Verify we captured everyone - self.assertIsNotNone(captured_num_choices, "Should have captured gathered_num_choices") - self.assertIsNotNone(captured_len_choices, "Should have captured gathered_len_choices") - self.assertIsNotNone(captured_len_context, "Should have captured gathered_len_context") - self.assertIsNotNone(captured_padded_logits, "Should have captured padded_logits_sums") - self.assertIsNotNone(captured_padded_max_equals, "Should have captured padded_max_equals") - self.assertIsNotNone(captured_padded_continuations, "Should have captured padded_continuations") - self.assertIsNotNone(captured_padded_contexts, "Should have captured padded_contexts") - - # Expected dimensions - expected_batch_size = len(docs) - expected_max_choices = max(len(doc.choices) for doc in docs) # Should be 5 - self.assertEqual(expected_max_choices, max(captured_num_choices)) - - # Test dimensions - # - Test num/len tensors - self.assertEqual(captured_num_choices.tolist(), [len(docs[0].choices), len(docs[1].choices)]) - self.assertEqual(captured_len_choices.tolist(), [2, 1]) # len of the choice, sample per sample, in tokens - self.assertEqual(captured_len_context.tolist(), [7, 6]) # len of the context, sample per sample, in tokens - - # - Test 1D padded tensors (logits_sums and max_equals) - self.assertEqual(captured_padded_logits.shape, (expected_batch_size, expected_max_choices)) - self.assertEqual(captured_padded_max_equals.shape, (expected_batch_size, expected_max_choices)) - - # - Test 3D padded continuations: (batch_size, max_num_choices, max_len_choices) - self.assertEqual( - captured_padded_continuations.shape, (len(docs), max(captured_num_choices), max(captured_len_choices)) - ) - - # - Test 2D padded contexts: (batch_size, max_len_context) - self.assertEqual(captured_padded_contexts.shape, (len(docs), max(captured_len_context))) - - # Verify padding values for 1D tensors - # - First doc has 3 choices, so positions [3:5] should be padded with -1 for logits and False for max_equals - self.assertTrue(torch.all(captured_padded_logits[0, 3:] == -1), "Padded positions in logits should be -1") - self.assertTrue( - torch.all(captured_padded_max_equals[0, 3:] == False), # noqa E712 - "Padded positions in max_equals should be False", - ) - - # - Second doc has 5 choices, so no padding needed for num_choices dimension - self.assertTrue( - torch.all(captured_padded_logits[1, :] != -1), "No padding should be needed for second doc logits" - ) - - # Verify padding values for 3D continuations tensor - # - First doc should have padding in the num_choices dimension (positions [3:5]) - self.assertTrue( - torch.all(captured_padded_continuations[0, 3:, :] == -1), "Padded choices in continuations should be -1" - ) - - # Verify padding values for 2D contexts tensor - # Both contexts should have some padding in the length dimension (padded with -1) - context_padding_mask = captured_padded_contexts == -1 - self.assertTrue(torch.any(context_padding_mask), "At least some context positions should be padded with -1") - - # Restore original gather function - self.model.accelerator.gather_for_metrics = lambda x: x - - -class TestTransformersModelUseChatTemplate(unittest.TestCase): - @patch("lighteval.models.transformers.transformers_model.Accelerator") - @patch("lighteval.models.transformers.transformers_model.TransformersModel._create_auto_model") - @patch("lighteval.utils.imports.is_package_available") - def test_transformers_model_use_chat_template_with_different_model_names( - self, mock_accelerator, mock_create_model, is_package_available - ): - """Test that TransformersModel correctly determines whether to use_chat_template or not automatically from the tokenizer config.""" - test_cases = [ - ("Qwen/Qwen3-0.6B", True), - ("gpt2", False), - ] - - for model_name, expected_result in test_cases: - with self.subTest(model_name=model_name): - # Mock accelerate related params - mock_accelerator_instance = Mock() - mock_accelerator_instance.device = torch.device("cpu") - mock_accelerator.return_value = mock_accelerator_instance - is_accelerate_available = False # noqa F841 - - # Skip the model creation phase - mock_create_model = Mock() # noqa F841 - - config = TransformersModelConfig(model_name=model_name, model_parallel=True, compile=False) - model = TransformersModel(config) - - self.assertEqual(model.use_chat_template, expected_result) - self.assertEqual(model.use_chat_template, model._tokenizer.chat_template is not None) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/models/vllm/test_vllm_model.py b/tests/unit/models/vllm/test_vllm_model.py deleted file mode 100644 index d4da6ce17..000000000 --- a/tests/unit/models/vllm/test_vllm_model.py +++ /dev/null @@ -1,62 +0,0 @@ -# MIT License - -# Copyright (c) 2025 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import unittest -from unittest.mock import Mock, patch - -from transformers import AutoTokenizer - -from lighteval.models.vllm.vllm_model import VLLMModel, VLLMModelConfig - - -class TestVLLMTokenizerCreation(unittest.TestCase): - def test_tokenizer_created_with_correct_revision(self): - config = VLLMModelConfig( - model_name="lighteval/different-chat-templates-per-revision", revision="new_chat_template" - ) - vllm_tokenizer = VLLMModel.__new__(VLLMModel)._create_auto_tokenizer(config) - tokenizer = AutoTokenizer.from_pretrained( - config.model_name, - revision=config.revision, - ) - self.assertEqual(vllm_tokenizer.chat_template, tokenizer.chat_template) - - -class TestVLLMModelUseChatTemplate(unittest.TestCase): - @patch("lighteval.models.vllm.vllm_model.VLLMModel._create_auto_model") - def test_vllm_model_use_chat_template_with_different_model_names(self, mock_create_model): - """Test that VLLMModel correctly calls uses_chat_template with different model names.""" - test_cases = [ - ("Qwen/Qwen3-0.6B", True), - ("gpt2", False), - ] - - for model_name, expected_result in test_cases: - with self.subTest(model_name=model_name): - # We skip the model creation phase - mock_create_model.return_value = Mock() - - config = VLLMModelConfig(model_name=model_name) - model = VLLMModel(config) - - self.assertEqual(model.use_chat_template, expected_result) - self.assertEqual(model.use_chat_template, model._tokenizer.chat_template is not None) diff --git a/tests/unit/pipeline/test_reasoning_tags.py b/tests/unit/pipeline/test_reasoning_tags.py deleted file mode 100644 index 603087ef9..000000000 --- a/tests/unit/pipeline/test_reasoning_tags.py +++ /dev/null @@ -1,415 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import tempfile -import unittest -from typing import Optional -from unittest.mock import patch - -from lighteval.logging.evaluation_tracker import EvaluationTracker -from lighteval.metrics.metrics import Metrics -from lighteval.models.dummy.dummy_model import DummyModel, DummyModelConfig -from lighteval.models.model_output import ModelResponse -from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters -from lighteval.tasks.lighteval_task import LightevalTask, LightevalTaskConfig -from lighteval.tasks.registry import Registry -from lighteval.tasks.requests import Doc, SamplingMethod -from lighteval.utils.imports import is_package_available - - -class TestPipelineReasoningTags(unittest.TestCase): - """Test suite for pipeline reasoning tags functionality using DummyModel.""" - - def setUp(self): - """Set up test fixtures.""" - self.temp_dir = tempfile.mkdtemp() - - # Create a simple test task - self.task_config = LightevalTaskConfig( - name="test_reasoning_task", - prompt_function=lambda x: x, - hf_repo="test_repo", - hf_subset="default", - metrics=[Metrics.exact_match], - hf_avail_splits=["test"], - evaluation_splits=["test"], - few_shots_split=None, - few_shots_select=None, - generation_size=10, - stop_sequence=["\n"], - num_fewshots=0, - ) - self.input_task_name = "test_reasoning_task|0" - self.task_config_name = self.task_config.full_name - - # Create test documents with reasoning tags in expected responses - self.test_docs = [ - Doc( - task_name=self.input_task_name, - query="What is 2+2?", - choices=["4"], - gold_index=[0], - instruction="", - sampling_methods=[SamplingMethod.GENERATIVE], - ), - ] - - # Mock dataset - self.mock_dataset = {"test": self.test_docs} - - def _mock_task_registry(self, input_task_name, task_config, task_docs, responses_with_reasoning_tags): - """Create a fake registry for testing.""" - - class FakeTask(LightevalTask): - def __post_init__(self): - self._docs = task_docs - - def get_docs(self, max_samples=None): - return task_docs - - @staticmethod - def download_dataset_worker(task) -> None: - # Mock dataset loading - return task._docs - - class FakeRegistry(Registry): - def __init__( - self, tasks: Optional[str] = None, load_multilingual: bool = False, custom_tasks: Optional[str] = None - ): - self.tasks_list = [input_task_name] - # suite_name, task_name, few_shot = input_task_name.split("|") - self.task_to_configs = {input_task_name: [task_config]} - - def load_tasks(self): - return {input_task_name: FakeTask(config=task_config)} - - # Create a DummyModel that returns responses with reasoning tags - class TestDummyModel(DummyModel): - def __init__(self, config): - super().__init__(config) - - def greedy_until(self, docs): - # Return responses with reasoning tags - return responses_with_reasoning_tags - - return FakeRegistry, TestDummyModel - - def test_remove_reasoning_tags_enabled(self): - """Test that reasoning tags are removed when remove_reasoning_tags=True.""" - - # Responses with reasoning tags - responses_with_reasoning = [ - ModelResponse(text=["Let me think about this... 2+2=4The answer is 4"]) - ] - - FakeRegistry, TestDummyModel = self._mock_task_registry( - self.input_task_name, self.task_config, self.test_docs, responses_with_reasoning - ) - - # Initialize accelerator if available - if is_package_available("accelerate"): - from accelerate import Accelerator - - Accelerator() - - with patch("lighteval.pipeline.Registry", FakeRegistry): - # Create pipeline with reasoning tag removal enabled - pipeline_params = PipelineParameters( - launcher_type=ParallelismManager.NONE, - remove_reasoning_tags=True, - reasoning_tags=[("", "")], - max_samples=1, - ) - - evaluation_tracker = EvaluationTracker(output_dir=self.temp_dir) - model = TestDummyModel(DummyModelConfig(seed=42)) - - pipeline = Pipeline( - tasks="test_reasoning_task|0", - pipeline_parameters=pipeline_params, - evaluation_tracker=evaluation_tracker, - model=model, - ) - - # Run the pipeline - pipeline.evaluate() - - # Check that reasoning tags were removed from post-processed text - details = pipeline.evaluation_tracker.details - self.assertEqual( - details["test_reasoning_task|0"][0]["model_response"]["text_post_processed"], ["The answer is 4"] - ) - - def test_remove_reasoning_tags_enabled_tags_as_string(self): - """Test that reasoning tags are removed when remove_reasoning_tags=True.""" - - # Responses with reasoning tags - responses_with_reasoning = [ - ModelResponse(text=["Let me think about this... 2+2=4The answer is 4"]) - ] - - FakeRegistry, TestDummyModel = self._mock_task_registry( - self.input_task_name, self.task_config, self.test_docs, responses_with_reasoning - ) - - # Initialize accelerator if available - if is_package_available("accelerate"): - from accelerate import Accelerator - - Accelerator() - - with patch("lighteval.pipeline.Registry", FakeRegistry): - # Create pipeline with reasoning tag removal enabled - pipeline_params = PipelineParameters( - launcher_type=ParallelismManager.NONE, - remove_reasoning_tags=True, - reasoning_tags='[("", "")]', - max_samples=1, - ) - - evaluation_tracker = EvaluationTracker(output_dir=self.temp_dir) - model = TestDummyModel(DummyModelConfig(seed=42)) - - pipeline = Pipeline( - tasks="test_reasoning_task|0", - pipeline_parameters=pipeline_params, - evaluation_tracker=evaluation_tracker, - model=model, - ) - - # Run the pipeline - pipeline.evaluate() - - # Check that reasoning tags were removed from post-processed text - details = pipeline.evaluation_tracker.details - self.assertEqual( - details["test_reasoning_task|0"][0]["model_response"]["text_post_processed"], ["The answer is 4"] - ) - - def test_remove_reasoning_tags_enabled_default_tags(self): - """Test that reasoning tags are removed when remove_reasoning_tags=True.""" - - # Responses with reasoning tags - responses_with_reasoning = [ - ModelResponse(text=["Let me think about this... 2+2=4The answer is 4"]) - ] - - FakeRegistry, TestDummyModel = self._mock_task_registry( - self.input_task_name, self.task_config, self.test_docs, responses_with_reasoning - ) - - # Initialize accelerator if available - if is_package_available("accelerate"): - from accelerate import Accelerator - - Accelerator() - - with patch("lighteval.pipeline.Registry", FakeRegistry): - # Create pipeline with reasoning tag removal enabled - pipeline_params = PipelineParameters( - launcher_type=ParallelismManager.NONE, remove_reasoning_tags=True, max_samples=1 - ) - - evaluation_tracker = EvaluationTracker(output_dir=self.temp_dir) - model = TestDummyModel(DummyModelConfig(seed=42)) - - pipeline = Pipeline( - tasks="test_reasoning_task|0", - pipeline_parameters=pipeline_params, - evaluation_tracker=evaluation_tracker, - model=model, - ) - - # Run the pipeline - pipeline.evaluate() - - # Check that reasoning tags were removed from post-processed text - details = pipeline.evaluation_tracker.details - self.assertEqual( - details["test_reasoning_task|0"][0]["model_response"]["text_post_processed"], ["The answer is 4"] - ) - - def test_remove_reasoning_tags_disabled(self): - """Test that reasoning tags are preserved when remove_reasoning_tags=False.""" - - # Responses with reasoning tags - responses_with_reasoning = [ - ModelResponse(text=["Let me think about this... 2+2=4The answer is 4"]) - ] - - FakeRegistry, TestDummyModel = self._mock_task_registry( - self.input_task_name, self.task_config, self.test_docs, responses_with_reasoning - ) - - # Initialize accelerator if available - if is_package_available("accelerate"): - from accelerate import Accelerator - - Accelerator() - - with patch("lighteval.pipeline.Registry", FakeRegistry): - # Create pipeline with reasoning tag removal disabled - pipeline_params = PipelineParameters( - launcher_type=ParallelismManager.NONE, - remove_reasoning_tags=False, - reasoning_tags=[("", "")], - max_samples=1, - ) - - evaluation_tracker = EvaluationTracker(output_dir=self.temp_dir) - model = TestDummyModel(DummyModelConfig(seed=42)) - - pipeline = Pipeline( - tasks="test_reasoning_task|0", - pipeline_parameters=pipeline_params, - evaluation_tracker=evaluation_tracker, - model=model, - ) - - # Run the pipeline - pipeline.evaluate() - - # Check that post-processed text is None (= no post processing happened) - details = pipeline.evaluation_tracker.details - self.assertIsNone( - details["test_reasoning_task|0"][0]["model_response"]["text_post_processed"], - ) - - def test_custom_reasoning_tags(self): - """Test that custom reasoning tags are correctly applied.""" - - # Responses with custom reasoning tags - responses_with_reasoning = [ - ModelResponse(text=["[reasoning]This is my thought process[/reasoning]Final answer: 4"]) - ] - - FakeRegistry, TestDummyModel = self._mock_task_registry( - self.input_task_name, self.task_config, self.test_docs, responses_with_reasoning - ) - - # Initialize accelerator if available - if is_package_available("accelerate"): - from accelerate import Accelerator - - Accelerator() - - with patch("lighteval.pipeline.Registry", FakeRegistry): - # Create pipeline with custom reasoning tags - pipeline_params = PipelineParameters( - launcher_type=ParallelismManager.NONE, - remove_reasoning_tags=True, - reasoning_tags=[("[reasoning]", "[/reasoning]")], - max_samples=1, - ) - - evaluation_tracker = EvaluationTracker(output_dir=self.temp_dir) - model = TestDummyModel(DummyModelConfig(seed=42)) - - pipeline = Pipeline( - tasks="test_reasoning_task|0", - pipeline_parameters=pipeline_params, - evaluation_tracker=evaluation_tracker, - model=model, - ) - - # Run the pipeline - pipeline.evaluate() - - # Check that reasoning tags were removed from post-processed text - details = pipeline.evaluation_tracker.details - self.assertEqual( - details["test_reasoning_task|0"][0]["model_response"]["text_post_processed"], ["Final answer: 4"] - ) - - def test_multiple_reasoning_tags(self): - """Test that multiple reasoning tag pairs are correctly handled.""" - - # Responses with multiple reasoning tag types - responses_with_reasoning = [ - ModelResponse(text=["First thoughtSome textSecond thoughtFinal: 4"]) - ] - - FakeRegistry, TestDummyModel = self._mock_task_registry( - self.input_task_name, self.task_config, self.test_docs, responses_with_reasoning - ) - - # Initialize accelerator if available - if is_package_available("accelerate"): - from accelerate import Accelerator - - Accelerator() - - with patch("lighteval.pipeline.Registry", FakeRegistry): - # Create pipeline with multiple reasoning tag pairs - pipeline_params = PipelineParameters( - launcher_type=ParallelismManager.NONE, - remove_reasoning_tags=True, - reasoning_tags='[("", ""), ("", "")]', - max_samples=1, - ) - - evaluation_tracker = EvaluationTracker(output_dir=self.temp_dir) - model = TestDummyModel(DummyModelConfig(seed=42)) - - pipeline = Pipeline( - tasks="test_reasoning_task|0", - pipeline_parameters=pipeline_params, - evaluation_tracker=evaluation_tracker, - model=model, - ) - - # Run the pipeline - pipeline.evaluate() - - # Check that reasoning tags were removed from post-processed text - details = pipeline.evaluation_tracker.details - self.assertEqual( - details["test_reasoning_task|0"][0]["model_response"]["text_post_processed"], - ["Some textFinal: 4"], - ) - - def test_reasoning_tags_validation(self): - """Test that invalid reasoning_tags parameter raises appropriate error.""" - - for test_string in ["['incorrect_format']", "invalid_format"]: - with self.assertRaises(ValueError) as context: - PipelineParameters( - launcher_type=ParallelismManager.NONE, - reasoning_tags=test_string, # Should be a list of tuples - ) - - # Check that the error message mentions the expected format - print(context.__dict__) - self.assertIn("reasoning_tags must be a list of pair tuples", str(context.exception)) - - def test_default_reasoning_tags(self): - """Test that default reasoning tags are correctly set.""" - - pipeline_params = PipelineParameters(launcher_type=ParallelismManager.NONE) - - # Check that default reasoning tags are set - self.assertEqual(pipeline_params.reasoning_tags, [("", "")]) - self.assertTrue(pipeline_params.remove_reasoning_tags) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/unit/prompt/test_prompt_manager.py b/tests/unit/prompt/test_prompt_manager.py deleted file mode 100644 index 6a0c6a326..000000000 --- a/tests/unit/prompt/test_prompt_manager.py +++ /dev/null @@ -1,59 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import random - -import pytest - -from lighteval.tasks.lighteval_task import LightevalTask, LightevalTaskConfig -from lighteval.tasks.prompt_manager import FewShotSampler -from lighteval.tasks.requests import Doc - - -@pytest.mark.parametrize("fewshot_select", ["sequential", "random", "balanced"]) -def test_fewshot_sampler(fewshot_select: str): - config = LightevalTaskConfig( - name="test_fewshot_task", - prompt_function=lambda _, __: None, - hf_repo="", - hf_subset="default", - metrics=[], - few_shots_split="test", - few_shots_select=fewshot_select, - ) - task = LightevalTask(config) - rnd = random.Random(0) - task._fewshot_docs = [ - Doc(str(i), ["A", "B"], rnd.randint(0, 2), fewshot_sorting_class=str(i % 20)) for i in range(100) - ] - sampler = FewShotSampler(task) - seed = 1 - docs = sampler.sample_fewshot_examples(20, seed) - - match task.fewshot_selection: - case "sequential": - assert docs == task.fewshot_docs()[:20] - case "random": - rnd = random.Random(seed) - task_docs = task.fewshot_docs() - rnd.shuffle(task_docs) - assert docs == task_docs[:20] diff --git a/tests/unit/prompt/test_prompt_manager_class.py b/tests/unit/prompt/test_prompt_manager_class.py deleted file mode 100644 index f552a9c31..000000000 --- a/tests/unit/prompt/test_prompt_manager_class.py +++ /dev/null @@ -1,463 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from unittest.mock import Mock - -import pytest - -from lighteval.tasks.prompt_manager import PromptManager -from lighteval.tasks.requests import Doc - - -class TestPromptManager: - """Test suite for the PromptManager class.""" - - def test_init_default_values(self): - """Test PromptManager initialization with default values.""" - pm = PromptManager() - assert pm.use_chat_template is False - assert pm.tokenizer is None - assert pm.system_prompt is None - - def test_init_with_chat_template(self): - """Test PromptManager initialization with chat template enabled.""" - tokenizer = Mock() - system_prompt = "You are a helpful assistant." - pm = PromptManager(use_chat_template=True, tokenizer=tokenizer, system_prompt=system_prompt) - assert pm.use_chat_template is True - assert pm.tokenizer == tokenizer - assert pm.system_prompt == system_prompt - - def test_prepare_prompt_plain_text_basic(self): - """Test prepare_prompt with plain text format and basic document.""" - pm = PromptManager() - doc = Doc(query="What is 2+2?", choices=["3", "4", "5"], gold_index=1) - - result = pm.prepare_prompt(doc) - assert result == "What is 2+2?" - - def test_prepare_prompt_plain_text_with_system_prompt(self): - """Test prepare_prompt with plain text format and system prompt.""" - pm = PromptManager(system_prompt="You are a math tutor.") - doc = Doc(query="What is 2+2?", choices=["3", "4", "5"], gold_index=1) - - result = pm.prepare_prompt(doc) - assert result == "You are a math tutor.\n\nWhat is 2+2?" - - def test_prepare_prompt_plain_text_with_instruction(self): - """Test prepare_prompt with plain text format and instruction.""" - pm = PromptManager() - doc = Doc( - query="What is 2+2?", - choices=["3", "4", "5"], - gold_index=1, - instruction="Please answer the following question:", - ) - - result = pm.prepare_prompt(doc) - assert result == "Please answer the following question:\n\nWhat is 2+2?" - - def test_prepare_prompt_plain_text_with_system_and_instruction(self): - """Test prepare_prompt with plain text format, system prompt and instruction.""" - pm = PromptManager(system_prompt="You are a math tutor.") - doc = Doc( - query="What is 2+2?", - choices=["3", "4", "5"], - gold_index=1, - instruction="Please answer the following question:", - ) - - result = pm.prepare_prompt(doc) - assert result == "You are a math tutor.\n\nPlease answer the following question:\n\nWhat is 2+2?" - - def test_prepare_prompt_plain_text_with_fewshot(self): - """Test prepare_prompt with plain text format and few-shot examples.""" - pm = PromptManager() - - # Create few-shot sample - fewshot_doc = Doc(query="What is 1+1?", choices=["1", "2", "3"], gold_index=1) - - doc = Doc(query="What is 2+2?", choices=["3", "4", "5"], gold_index=1) - doc.fewshot_samples = [fewshot_doc] - - result = pm.prepare_prompt(doc) - assert result == "What is 1+1? 2\n\nWhat is 2+2?" - - def test_prepare_prompt_plain_text_with_fewshot_and_instruction(self): - """Test prepare_prompt with plain text format, few-shot examples and instruction.""" - pm = PromptManager() - - # Create few-shot sample with instruction - fewshot_doc = Doc( - query="Please answer the following question: What is 1+1?", - choices=["1", "2", "3"], - gold_index=1, - instruction="Please answer the following question:", - ) - - doc = Doc( - query="Please answer the following question: What is 2+2?", - choices=["3", "4", "5"], - gold_index=1, - instruction="Please answer the following question:", - ) - doc.fewshot_samples = [fewshot_doc] - - result = pm.prepare_prompt(doc) - assert result == "Please answer the following question:\n\nWhat is 1+1? 2\n\nWhat is 2+2?" - - def test_prepare_prompt_chat_template_basic(self): - """Test prepare_prompt with chat template format and basic document.""" - tokenizer = Mock() - tokenizer.apply_chat_template.return_value = "<|user|>\nWhat is 2+2?<|assistant|>" - - pm = PromptManager(use_chat_template=True, tokenizer=tokenizer) - doc = Doc(query="What is 2+2?", choices=["3", "4", "5"], gold_index=1) - - result = pm.prepare_prompt(doc) - assert result == "<|user|>\nWhat is 2+2?<|assistant|>" - tokenizer.apply_chat_template.assert_called_once() - - def test_prepare_prompt_chat_template_with_system_prompt(self): - """Test prepare_prompt with chat template format and system prompt.""" - tokenizer = Mock() - tokenizer.apply_chat_template.return_value = ( - "<|system|>\nYou are a math tutor.<|user|>\nWhat is 2+2?<|assistant|>" - ) - - pm = PromptManager(use_chat_template=True, tokenizer=tokenizer, system_prompt="You are a math tutor.") - doc = Doc(query="What is 2+2?", choices=["3", "4", "5"], gold_index=1) - - result = pm.prepare_prompt(doc) - assert result == "<|system|>\nYou are a math tutor.<|user|>\nWhat is 2+2?<|assistant|>" - - # Verify the call arguments - call_args = tokenizer.apply_chat_template.call_args - messages = call_args[0][0] - assert len(messages) == 2 - assert messages[0]["role"] == "system" - assert messages[0]["content"] == "You are a math tutor." - assert messages[1]["role"] == "user" - assert messages[1]["content"] == "What is 2+2?" - - def test_prepare_prompt_chat_template_with_fewshot(self): - """Test prepare_prompt with chat template format and few-shot examples.""" - tokenizer = Mock() - tokenizer.apply_chat_template.return_value = ( - "<|user|>\nWhat is 1+1?<|assistant|>\n2<|user|>\nWhat is 2+2?<|assistant|>" - ) - - pm = PromptManager(use_chat_template=True, tokenizer=tokenizer) - - # Create few-shot sample - fewshot_doc = Doc(query="What is 1+1?", choices=["1", "2", "3"], gold_index=1) - - doc = Doc(query="What is 2+2?", choices=["3", "4", "5"], gold_index=1) - doc.fewshot_samples = [fewshot_doc] - - result = pm.prepare_prompt(doc) - assert result == "<|user|>\nWhat is 1+1?<|assistant|>\n2<|user|>\nWhat is 2+2?<|assistant|>" - - # Verify the call arguments - call_args = tokenizer.apply_chat_template.call_args - messages = call_args[0][0] - assert len(messages) == 3 - assert messages[0]["role"] == "user" - assert messages[0]["content"] == "What is 1+1?" - assert messages[1]["role"] == "assistant" - assert messages[1]["content"] == "2" - assert messages[2]["role"] == "user" - assert messages[2]["content"] == "What is 2+2?" - - def test_prepare_prompt_chat_template_with_fewshot_and_instruction(self): - """Test prepare_prompt with chat template format, few-shot examples and instruction.""" - tokenizer = Mock() - tokenizer.apply_chat_template.return_value = "<|user|>\nPlease answer the following question: What is 1+1?<|assistant|>\n2<|user|>\nWhat is 2+2?<|assistant|>" - - pm = PromptManager(use_chat_template=True, tokenizer=tokenizer) - - # Create few-shot sample with instruction - fewshot_doc = Doc( - query="Please answer the following question: What is 1+1?", - choices=["1", "2", "3"], - gold_index=1, - instruction="Please answer the following question: ", - ) - - doc = Doc( - query="Please answer the following question: What is 2+2?", - choices=["3", "4", "5"], - gold_index=1, - instruction="Please answer the following question: ", - ) - doc.fewshot_samples = [fewshot_doc] - - result = pm.prepare_prompt(doc) - assert ( - result - == "<|user|>\nPlease answer the following question: What is 1+1?<|assistant|>\n2<|user|>\nWhat is 2+2?<|assistant|>" - ) - - # Verify the call arguments - call_args = tokenizer.apply_chat_template.call_args - messages = call_args[0][0] - assert len(messages) == 3 - assert messages[0]["role"] == "user" - assert messages[0]["content"] == "Please answer the following question: What is 1+1?" - assert messages[1]["role"] == "assistant" - assert messages[1]["content"] == "2" - assert messages[2]["role"] == "user" - assert messages[2]["content"] == "What is 2+2?" - - def test_prepare_prompt_chat_template_with_fewshot_and_instruction_and_system_prompt(self): - """Test prepare_prompt with chat template format, few-shot examples, instruction and system prompt.""" - tokenizer = Mock() - tokenizer.apply_chat_template.return_value = "<|system|>You are a math tutor.\nPlease answer the following question:<|user|>\nWhat is 1+1?<|assistant|>\n2<|user|>\nWhat is 2+2?<|assistant|>" - - pm = PromptManager(use_chat_template=True, tokenizer=tokenizer, system_prompt="You are a math tutor.") - - # Create few-shot sample with instruction - fewshot_doc = Doc( - query="Please answer the following question: What is 1+1?", - choices=["1", "2", "3"], - gold_index=1, - instruction="Please answer the following question:", - ) - - doc = Doc( - query="Please answer the following question: What is 2+2?", - choices=["3", "4", "5"], - gold_index=1, - instruction="Please answer the following question:", - ) - doc.fewshot_samples = [fewshot_doc] - - result = pm.prepare_prompt(doc) - assert ( - result - == "<|system|>You are a math tutor.\nPlease answer the following question:<|user|>\nWhat is 1+1?<|assistant|>\n2<|user|>\nWhat is 2+2?<|assistant|>" - ) - - # Verify the call arguments - call_args = tokenizer.apply_chat_template.call_args - messages = call_args[0][0] - assert len(messages) == 4 - - def test_prepare_prompt_chat_template_no_tokenizer(self): - """Test prepare_prompt with chat template but no tokenizer raises error.""" - pm = PromptManager(use_chat_template=True, tokenizer=None) - doc = Doc(query="What is 2+2?", choices=["3", "4", "5"], gold_index=1) - - with pytest.raises(AssertionError, match="Tokenizer must be set for chat template formatting."): - pm.prepare_prompt(doc) - - def test_prepare_prompt_api_basic(self): - """Test prepare_prompt_api with basic document.""" - pm = PromptManager() - doc = Doc(query="What is 2+2?", choices=["3", "4", "5"], gold_index=1) - - result = pm.prepare_prompt_api(doc) - assert result == [{"role": "user", "content": "What is 2+2?"}] - - def test_prepare_prompt_api_with_system_prompt(self): - """Test prepare_prompt_api with system prompt.""" - pm = PromptManager(system_prompt="You are a math tutor.") - doc = Doc(query="What is 2+2?", choices=["3", "4", "5"], gold_index=1) - - result = pm.prepare_prompt_api(doc) - assert result == [ - {"role": "system", "content": "You are a math tutor."}, - {"role": "user", "content": "What is 2+2?"}, - ] - - def test_prepare_prompt_api_with_instruction(self): - """Test prepare_prompt_api with instruction.""" - pm = PromptManager() - doc = Doc( - query="What is 2+2?", - choices=["3", "4", "5"], - gold_index=1, - instruction="Please answer the following question: ", - ) - - result = pm.prepare_prompt_api(doc) - - assert result == [ - {"role": "user", "content": "Please answer the following question: What is 2+2?"}, - ] - - def test_prepare_prompt_api_with_system_and_instruction(self): - """Test prepare_prompt_api with system prompt and instruction.""" - pm = PromptManager(system_prompt="You are a math tutor.") - doc = Doc( - query="What is 2+2?", - choices=["3", "4", "5"], - gold_index=1, - instruction="Please answer the following question: ", - ) - - result = pm.prepare_prompt_api(doc) - assert result == [ - {"role": "system", "content": "You are a math tutor."}, - {"role": "user", "content": "Please answer the following question: What is 2+2?"}, - ] - - def test_prepare_prompt_multimodal_basic(self): - """Test prepare_prompt_multimodal with basic multimodal document.""" - tokenizer = Mock() - tokenizer.apply_chat_template.return_value = "<|user|>\nWhat is in this image?<|assistant|>" - - pm = PromptManager(use_chat_template=True, tokenizer=tokenizer) - - # Mock image - mock_image = Mock() - - doc = Doc( - query="What is in this image?", choices=["A cat", "A dog", "A bird"], gold_index=0, images=[mock_image] - ) - - result = pm.prepare_prompt_multimodal(doc) - assert result == "<|user|>\nWhat is in this image?<|assistant|>" - - # Verify the call arguments - call_args = tokenizer.apply_chat_template.call_args - messages = call_args[0][0] - assert len(messages) == 1 - assert messages[0]["role"] == "user" - assert len(messages[0]["content"]) == 2 - assert messages[0]["content"][0]["type"] == "text" - assert messages[0]["content"][0]["text"] == "What is in this image?" - assert messages[0]["content"][1]["type"] == "image" - assert messages[0]["content"][1]["image"] == mock_image - - def test_prepare_prompt_multimodal_with_system_prompt(self): - """Test prepare_prompt_multimodal with system prompt.""" - tokenizer = Mock() - tokenizer.apply_chat_template.return_value = ( - "<|system|>\nYou are a helpful assistant.<|user|>\nWhat is in this image?<|assistant|>" - ) - - pm = PromptManager(use_chat_template=True, tokenizer=tokenizer, system_prompt="You are a helpful assistant.") - - mock_image = Mock() - doc = Doc( - query="What is in this image?", choices=["A cat", "A dog", "A bird"], gold_index=0, images=[mock_image] - ) - - result = pm.prepare_prompt_multimodal(doc) - assert result == "<|system|>\nYou are a helpful assistant.<|user|>\nWhat is in this image?<|assistant|>" - - # Verify the call arguments - call_args = tokenizer.apply_chat_template.call_args - messages = call_args[0][0] - assert len(messages) == 2 - assert messages[0]["role"] == "system" - assert messages[0]["content"][0]["type"] == "text" - assert messages[0]["content"][0]["text"] == "You are a helpful assistant." - assert messages[1]["role"] == "user" - assert len(messages[1]["content"]) == 2 - - def test_prepare_prompt_multimodal_no_chat_template(self): - """Test prepare_prompt_multimodal without chat template raises error.""" - pm = PromptManager(use_chat_template=False) - mock_image = Mock() - doc = Doc( - query="What is in this image?", choices=["A cat", "A dog", "A bird"], gold_index=0, images=[mock_image] - ) - - with pytest.raises(ValueError, match="Multimodal prompts are only supported with chat template format."): - pm.prepare_prompt_multimodal(doc) - - def test_prepare_prompt_multimodal_no_tokenizer(self): - """Test prepare_prompt_multimodal without tokenizer raises error.""" - pm = PromptManager(use_chat_template=True, tokenizer=None) - mock_image = Mock() - doc = Doc( - query="What is in this image?", choices=["A cat", "A dog", "A bird"], gold_index=0, images=[mock_image] - ) - - with pytest.raises(ValueError, match="Multimodal prompts are only supported with chat template format."): - pm.prepare_prompt_multimodal(doc) - - def test_prepare_prompt_multimodal_no_images(self): - """Test prepare_prompt_multimodal without images raises error.""" - tokenizer = Mock() - pm = PromptManager(use_chat_template=True, tokenizer=tokenizer) - doc = Doc(query="What is in this image?", choices=["A cat", "A dog", "A bird"], gold_index=0, images=None) - - with pytest.raises(ValueError, match="Multimodal prompts require images to be provided in the document."): - pm.prepare_prompt_multimodal(doc) - - def test_extract_query_no_instruction(self): - """Test _extract_query with no instruction.""" - pm = PromptManager() - result = pm._extract_query("What is 2+2?", None) - assert result == "What is 2+2?" - - def test_extract_query_with_instruction(self): - """Test _extract_query with instruction but no previous shots.""" - pm = PromptManager() - result = pm._extract_query("Please answer: What is 2+2?", "Please answer: ") - assert result == "What is 2+2?" - - def test_prepare_prompt_with_multiple_fewshot_examples(self): - """Test prepare_prompt with multiple few-shot examples.""" - pm = PromptManager() - - # Create few-shot samples - fewshot_doc1 = Doc(query="What is 1+1?", choices=["1", "2", "3"], gold_index=1) - fewshot_doc2 = Doc(query="What is 3+3?", choices=["5", "6", "7"], gold_index=1) - - doc = Doc(query="What is 2+2?", choices=["3", "4", "5"], gold_index=1) - doc.fewshot_samples = [fewshot_doc1, fewshot_doc2] - - result = pm.prepare_prompt(doc) - assert result == "What is 1+1? 2\n\nWhat is 3+3? 6\n\nWhat is 2+2?" - - def test_prepare_prompt_with_empty_fewshot_samples(self): - """Test prepare_prompt with empty few-shot samples.""" - pm = PromptManager() - doc = Doc(query="What is 2+2?", choices=["3", "4", "5"], gold_index=1) - doc.fewshot_samples = [] - - result = pm.prepare_prompt(doc) - assert result == "What is 2+2?" - - def test_prepare_prompt_with_complex_instruction_removal(self): - """Test prepare_prompt with complex instruction removal in few-shot examples.""" - pm = PromptManager() - - # Create few-shot sample with complex instruction - instruction = "Please provide a detailed mathematical answer to the following question:" - fewshot_doc = Doc(query="What is 1+1?", choices=["1", "2", "3"], gold_index=1, instruction=instruction) - - doc = Doc( - query="What is 2+2?", - choices=["3", "4", "5"], - gold_index=1, - instruction=instruction, - ) - doc.fewshot_samples = [fewshot_doc] - - result = pm.prepare_prompt(doc) - expected = f"{instruction}\n\nWhat is 1+1? 2\n\nWhat is 2+2?" - assert result == expected diff --git a/tests/unit/tasks/templates/test_continuation.py b/tests/unit/tasks/templates/test_continuation.py deleted file mode 100644 index 9681ecd66..000000000 --- a/tests/unit/tasks/templates/test_continuation.py +++ /dev/null @@ -1,144 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from lighteval.tasks.templates.continuation import get_continuation_prompt_function -from lighteval.tasks.templates.utils.formulation import CFFormulation, MCFFormulation -from lighteval.utils.language import Language - - -def test_continuation_prompt_mcf(): - """Test multiple-choice format continuation prompt generation.""" - test_input = { - "context": "The quick brown fox", - "continuations": ["jumps over the lazy dog", "Runs through the forest", "Chases a rabbit"], - "gold_idx": 0, - } - - prompt_fn = get_continuation_prompt_function( - Language.ENGLISH, - {"context": "context", "continuations": "continuations", "gold_idx": "gold_idx"}, - MCFFormulation(), - ) - - doc = prompt_fn(test_input, "test_continuation_task") - - # We expect the contuation to be decapitalized as it's continuation of non-ended sentence - assert ( - doc.query - == """\ -The quick brown fox - A. jumps over the lazy dog - B. runs through the forest - C. chases a rabbit -Answer:\ -""" - ) - - assert doc.unconditioned_query == "Answer:" - assert doc.choices == [" A", " B", " C"] - assert doc.gold_index == [0] - - -def test_continuation_prompt_cf(): - """Test cloze format continuation prompt generation.""" - test_input = { - "context": "The sun is", - "continuations": ["shining brightly", "setting in the west", "hidden behind clouds"], - "gold_idx": 1, - } - - prompt_fn = get_continuation_prompt_function( - Language.ENGLISH, - {"context": "context", "continuations": "continuations", "gold_idx": "gold_idx"}, - CFFormulation(), - ) - - doc = prompt_fn(test_input, "test_continuation_task") - - assert doc.query == "The sun is" - - assert doc.unconditioned_query == "" - assert doc.choices == [" shining brightly", " setting in the west", " hidden behind clouds"] - assert doc.gold_index == [1] - - -def test_continuation_prompt_sequence_end(): - """Test continuation prompt generation when the context ends with a period.""" - test_input = { - "context": "the sun is.", - "continuations": ["shining brightly", "setting in the west", "hidden behind clouds"], - "gold_idx": 1, - } - - prompt_fn = get_continuation_prompt_function( - Language.ENGLISH, - {"context": "context", "continuations": "continuations", "gold_idx": "gold_idx"}, - CFFormulation(), - ) - - doc = prompt_fn(test_input, "test_continuation_task") - - assert doc.query == "The sun is." - - # Here we expect the continuation to be capitalized as it's a standalone sentence - assert doc.unconditioned_query == "" - assert doc.choices == [" Shining brightly", " Setting in the west", " Hidden behind clouds"] - assert doc.gold_index == [1] - - -def test_continuation_optional_keys(): - """Test continuation prompt generation with optional instruction key.""" - test_input = { - "context": "In the morning, I like to", - "continuations": ["drink coffee", "go for a run", "read the news"], - "gold_idx": 0, - "instruction": "Choose the most likely continuation:", - } - - prompt_fn = get_continuation_prompt_function( - Language.ENGLISH, - { - "context": "context", - "continuations": "continuations", - "gold_idx": "gold_idx", - "instruction": "instruction", - }, - MCFFormulation(), - ) - - doc = prompt_fn(test_input, "test_continuation_task") - - assert ( - doc.query - == """\ -Choose the most likely continuation: -In the morning, I like to - A. drink coffee - B. go for a run - C. read the news -Answer:\ -""" - ) - - assert doc.unconditioned_query == "Answer:" - assert doc.choices == [" A", " B", " C"] - assert doc.gold_index == [0] diff --git a/tests/unit/tasks/templates/test_copa.py b/tests/unit/tasks/templates/test_copa.py deleted file mode 100644 index 775fb37fb..000000000 --- a/tests/unit/tasks/templates/test_copa.py +++ /dev/null @@ -1,62 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import pytest - -from lighteval.tasks.templates.copa import get_copa_prompt_function -from lighteval.tasks.templates.utils.formulation import CFFormulation -from lighteval.utils.language import Language - - -@pytest.mark.parametrize("cause_effect", ["cause", "effect"]) -def test_copa_prompt_cf(cause_effect): - """ - Tests that copa prompt function works correctly for both cause/effect. - Since it's pretty much a wrapper around continuation template we just test single formulation. - - """ - test_input = { - "cause_effect": cause_effect, - "context": "He is strong", - "continuations": ["he has big muscles", "he is weak"], - "gold_idx": 0, - } - - prompt_fn = get_copa_prompt_function( - Language.ENGLISH, - { - "cause_effect": "cause_effect", - "context": "context", - "continuations": "continuations", - "gold_idx": "gold_idx", - }, - CFFormulation(), - ) - - doc = prompt_fn(test_input, "test_task") - - cause_effect_word = "because" if cause_effect == "cause" else "therefore" - assert doc.query == f"He is strong {cause_effect_word}" - - assert doc.unconditioned_query == "" - assert doc.choices == [" he has big muscles", " he is weak"] - assert doc.gold_index == [0] diff --git a/tests/unit/tasks/templates/test_hellaswag.py b/tests/unit/tasks/templates/test_hellaswag.py deleted file mode 100644 index 2ef7b895b..000000000 --- a/tests/unit/tasks/templates/test_hellaswag.py +++ /dev/null @@ -1,160 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -from lighteval.tasks.templates.hellaswag import get_hellaswag_prompt_function -from lighteval.tasks.templates.utils.formulation import CFFormulation, MCFFormulation -from lighteval.utils.language import Language - - -def test_hellaswag_prompt_cf(): - """ - Tests that hellaswag prompt function works correctly. - Since it's pretty much a wrapper around continuation template we just test single formulation. - - """ - test_input = { - "activity_label": "fitness", - "ctx_a": "He is strong", - "ctx_b": "He is fast", - "continuations": ["he has big muscles", "he is weak"], - "gold_idx": 0, - } - - prompt_fn = get_hellaswag_prompt_function( - Language.ENGLISH, - { - "activity_label": "activity_label", - "continuations": "continuations", - "gold_idx": "gold_idx", - "ctx_a": "ctx_a", - "ctx_b": "ctx_b", - }, - CFFormulation(), - ) - - doc = prompt_fn(test_input, "test_task") - assert doc.query == "Fitness:\nHe is strong he is fast" - - assert doc.unconditioned_query == "" - assert doc.choices == [" he has big muscles", " he is weak"] - assert doc.gold_index == [0] - - -def test_hellaswag_prompt_mcf(): - """ - Tests that hellaswag prompt function works correctly. - Since it's pretty much a wrapper around continuation template we just test single formulation. - - """ - test_input = { - "activity_label": "fitness", - "ctx_a": "He is strong", - "ctx_b": "He is fast", - "continuations": ["he has big muscles", "he is weak"], - "gold_idx": 0, - } - - prompt_fn = get_hellaswag_prompt_function( - Language.ENGLISH, - { - "activity_label": "activity_label", - "continuations": "continuations", - "gold_idx": "gold_idx", - "ctx_a": "ctx_a", - "ctx_b": "ctx_b", - }, - MCFFormulation(), - ) - - doc = prompt_fn(test_input, "test_task") - assert ( - doc.query - == """\ -Fitness:\nHe is strong he is fast - A. he has big muscles - B. he is weak -Answer:\ -""" - ) - - assert doc.unconditioned_query == "Answer:" - assert doc.choices == [" A", " B"] - assert doc.gold_index == [0] - - -def test_hellaswag_ctx_joining(): - """ - Tests that hellaswag prompt function works correctly. - Since it's pretty much a wrapper around continuation template we just test single formulation. - - """ - test_input = { - "activity_label": "fitness", - "ctx_a": "He is strong.", - "ctx_b": "he is fast.", - "continuations": ["he has big muscles", "he is weak"], - "gold_idx": 0, - } - - prompt_fn = get_hellaswag_prompt_function( - Language.ENGLISH, - { - "activity_label": "activity_label", - "continuations": "continuations", - "gold_idx": "gold_idx", - "ctx_a": "ctx_a", - "ctx_b": "ctx_b", - }, - CFFormulation(), - ) - - doc = prompt_fn(test_input, "test_task") - assert doc.query == "Fitness:\nHe is strong. He is fast." - - -def test_hellaswag_single_ctx(): - """ - Tests that hellaswag prompt function works correctly. - Since it's pretty much a wrapper around continuation template we just test single formulation. - - """ - test_input = { - "activity_label": "fitness", - "ctx_a": "He is strong.", - "continuations": ["he has big muscles", "he is weak"], - "gold_idx": 0, - } - - prompt_fn = get_hellaswag_prompt_function( - Language.ENGLISH, - { - "activity_label": "activity_label", - "continuations": "continuations", - "gold_idx": "gold_idx", - "ctx_a": "ctx_a", - }, - CFFormulation(), - ) - - doc = prompt_fn(test_input, "test_task") - assert doc.query == "Fitness:\nHe is strong." diff --git a/tests/unit/tasks/templates/test_multichoice.py b/tests/unit/tasks/templates/test_multichoice.py deleted file mode 100644 index 9b3614ed2..000000000 --- a/tests/unit/tasks/templates/test_multichoice.py +++ /dev/null @@ -1,188 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from lighteval.tasks.templates.multichoice import get_mcq_prompt_function -from lighteval.tasks.templates.utils.formulation import CFFormulation, MCFFormulation -from lighteval.utils.language import Language - - -def test_multichoice_prompt_mcf(): - """Test multiple-choice format (MCF) prompt generation for multichoice questions.""" - test_input = { - "question": "What is the capital of France?", - "choices": ["London", "Paris", "Berlin", "Madrid"], - "gold_idx": 1, - } - - prompt_fn = get_mcq_prompt_function( - Language.ENGLISH, - { - "question": "question", - "choices": "choices", - "gold_idx": "gold_idx", - }, - MCFFormulation(), - ) - - doc = prompt_fn(test_input, "test_task") - - assert ( - doc.query - == """\ -Question: What is the capital of France? - A. London - B. Paris - C. Berlin - D. Madrid -Answer:\ -""" - ) - - assert doc.unconditioned_query == "Answer:" - assert doc.choices == [" A", " B", " C", " D"] - - -def test_multichoice_prompt_nli_cf(): - """Test cloze format (CF) prompt generation for multichoice questions.""" - test_input = { - "question": "What is the capital of France?", - "choices": ["London", "Paris", "Berlin", "Madrid"], - "gold_idx": 1, - } - - prompt_fn = get_mcq_prompt_function( - Language.ENGLISH, - { - "question": "question", - "choices": "choices", - "gold_idx": "gold_idx", - }, - CFFormulation(), - ) - doc = prompt_fn(test_input, "test_task") - - assert ( - doc.query - == """\ -Question: What is the capital of France? -Answer:\ -""" - ) - - assert doc.unconditioned_query == "Answer:" - assert doc.choices == [" London", " Paris", " Berlin", " Madrid"] - - -def test_chinese_multichoice_prompt(): - """Test multichoice prompt generation for Chinese language.""" - test_input = { - "question": "什么是中国的首都?", - "choices": ["北京", "上海", "广州", "深圳"], - "gold_idx": 0, - } - - prompt_fn = get_mcq_prompt_function( - Language.CHINESE, {"question": "question", "choices": "choices", "gold_idx": "gold_idx"}, MCFFormulation() - ) - - doc = prompt_fn(test_input, "test_task") - - assert ( - doc.query - == """\ -问题：什么是中国的首都？ -A。北京 -B。上海 -C。广州 -D。深圳 -答案：\ -""" - ) - - -def test_thai_multichoice_prompt(): - """Test multichoice prompt generation for Thai language.""" - test_input = { - "question": "สิ่งใดต่อไปนี้เป็นสิ่งที่คุณชอบมากที่สุด?", - "choices": ["รถยนต์", "รถจักรยานยนต์", "รถจักรยานยนต์", "รถยนต์"], - "gold_idx": 0, - } - - prompt_fn = get_mcq_prompt_function( - Language.THAI, {"question": "question", "choices": "choices", "gold_idx": "gold_idx"}, MCFFormulation() - ) - - doc = prompt_fn(test_input, "test_task") - - assert ( - doc.query - == """\ -คำถาม: สิ่งใดต่อไปนี้เป็นสิ่งที่คุณชอบมากที่สุด? - A. รถยนต์ - B. รถจักรยานยนต์ - C. รถจักรยานยนต์ - D. รถยนต์ -คำตอบ:\ -""" - ) - - assert doc.unconditioned_query == "คำตอบ:" - assert doc.choices == [" A", " B", " C", " D"] - - -def test_multichoice_optional_keys(): - """Test multichoice prompt generation with optional keys (context and instruction).""" - test_input = { - "question": "What is the capital of France?", - "choices": ["London", "Paris", "Berlin", "Madrid"], - "context": "France is big.", - "instruction": "Please answer the following question about geography.", - "gold_idx": 1, - } - - prompt_fn = get_mcq_prompt_function( - Language.ENGLISH, - { - "question": "question", - "choices": "choices", - "gold_idx": "gold_idx", - "context": "context", - "instruction": "instruction", - }, - MCFFormulation(), - ) - - doc = prompt_fn(test_input, "test_task") - - assert ( - doc.query - == """\ -Please answer the following question about geography. -France is big. -Question: What is the capital of France? - A. London - B. Paris - C. Berlin - D. Madrid -Answer:\ -""" - ) diff --git a/tests/unit/tasks/templates/test_nli.py b/tests/unit/tasks/templates/test_nli.py deleted file mode 100644 index a634432b9..000000000 --- a/tests/unit/tasks/templates/test_nli.py +++ /dev/null @@ -1,116 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from lighteval.tasks.templates.nli import get_nli_prompt_function -from lighteval.tasks.templates.utils.formulation import CFFormulation, HybridFormulation -from lighteval.utils.language import Language - - -def test_nli_prompt_mcf(): - """Test multiple-choice format NLI prompt generation.""" - test_input = { - "premise": "The cat is sleeping on the couch.", - "hypothesis": "The cat is awake.", - "gold_idx": 2, - } - - prompt_fn = get_nli_prompt_function( - Language.ENGLISH, - {"hypothesis": "hypothesis", "premise": "premise", "gold_idx": "gold_idx"}, - ["entailment", "neutral", "contradiction"], - ) - - doc = prompt_fn(test_input, "test_nli_task") - - assert ( - doc.query - == """\ -The cat is sleeping on the couch. -Question: The cat is awake. - A. True - B. Neither - C. False -Answer:\ -""" - ) - assert doc.unconditioned_query == "Answer:" - assert doc.choices == [" A", " B", " C"] - assert doc.gold_index == [2] - - -def test_nli_prompt_cf(): - """Test cloze format NLI prompt generation.""" - test_input = { - "premise": "The cat is sleeping on the couch.", - "hypothesis": "The cat is awake.", - "gold_idx": 2, - } - - prompt_fn = get_nli_prompt_function( - Language.ENGLISH, - {"hypothesis": "hypothesis", "premise": "premise", "gold_idx": "gold_idx"}, - ["entailment", "neutral", "contradiction"], - formulation=CFFormulation(), - ) - - doc = prompt_fn(test_input, "test_nli_task") - - assert doc.query == "The cat is sleeping on the couch right?" - assert doc.unconditioned_query == "right?" - assert doc.choices == [" Yes, the cat is awake", " Also, the cat is awake", " No, the cat is awake"] - assert doc.gold_index == 2 - - test_input = { - "premise": "The cat is sleeping on the couch.", - "hypothesis": "The cat is awake.", - "gold_idx": 2, - } - - -def test_nli_prompt_hybrid(): - """Test hybrid format NLI prompt generation.""" - - test_input = { - "premise": "The cat is sleeping on the couch.", - "hypothesis": "The cat is awake.", - "gold_idx": 2, - } - prompt_fn = get_nli_prompt_function( - Language.ENGLISH, - {"hypothesis": "hypothesis", "premise": "premise", "gold_idx": "gold_idx"}, - ["entailment", "neutral", "contradiction"], - formulation=HybridFormulation(), - ) - - doc = prompt_fn(test_input, "test_nli_task") - - assert ( - doc.query - == """\ -The cat is sleeping on the couch. -Question: The cat is awake True, False or Neither? -Answer:\ -""" - ) - assert doc.unconditioned_query == "Answer:" - assert doc.choices == [" True", " Neither", " False"] - assert doc.gold_index == [2] diff --git a/tests/unit/tasks/templates/test_translation.py b/tests/unit/tasks/templates/test_translation.py deleted file mode 100644 index eab59cf18..000000000 --- a/tests/unit/tasks/templates/test_translation.py +++ /dev/null @@ -1,120 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -from lighteval.tasks.templates.translation import get_translation_prompt_function -from lighteval.tasks.templates.utils.formulation import CFFormulation, MCFFormulation -from lighteval.utils.language import Language - - -def test_translation_prompt_cf(): - """ - Tests that translation prompt function works correctly for CF formulation. - """ - test_input = { - "source_text": "Ahoj, jak se máš?", - "target_text": "Bonjour, comment allez-vous?", - } - - prompt_fn = get_translation_prompt_function( - source_language=Language.CZECH, - target_language=Language.FRENCH, - adapter=lambda x: { - "source_text": x["source_text"], - "target_text": x["target_text"], - }, - formulation=CFFormulation(), - ) - - doc = prompt_fn(test_input, "test_task") - assert doc is not None - - assert doc.query == "CS: Ahoj, jak se máš? FR:" - assert doc.unconditioned_query == "" - assert doc.choices == [" Bonjour, comment allez-vous?"] - assert doc.gold_index == [0] - - -def test_translation_prompt_mcf(): - """ - Tests that translation prompt function works correctly for MCF formulation. - """ - test_input = { - "source_text": "Ahoj, jak se máš?", - "target_text": ["Bonjour, comment allez-vous?", "Ciao, come stai?"], - } - - prompt_fn = get_translation_prompt_function( - source_language=Language.CZECH, - target_language=Language.FRENCH, - adapter=lambda x: { - "source_text": x["source_text"], - "target_text": x["target_text"], - "gold_idx": 0, - }, - formulation=MCFFormulation(), - ) - - doc = prompt_fn(test_input, "test_task") - assert doc is not None - - assert ( - doc.query - == """\ -CS: Ahoj, jak se máš? FR: - A. Bonjour, comment allez-vous? - B. Ciao, come stai? -Answer:\ -""" - ) - assert doc.unconditioned_query == "Answer:" - assert doc.choices == [" A", " B"] - assert doc.gold_index == [0] - - -def test_translation_prompt_cf_formatting(): - """ - Tests that translation prompt function works correctly for CF formulation with formatting. - """ - test_input = { - "source_text": "How are you?", - "target_text": ["你好吗?"], - } - - prompt_fn = get_translation_prompt_function( - source_language=Language.ENGLISH, - target_language=Language.CHINESE, - adapter=lambda x: { - "source_text": x["source_text"], - "target_text": x["target_text"], - "gold_idx": 0, - }, - formulation=CFFormulation(), - ) - - doc = prompt_fn(test_input, "test_task") - assert doc is not None - - assert doc.query == "EN: How are you? ZH:" - assert doc.unconditioned_query == "" - assert doc.choices == [" 你好吗？"] - assert doc.gold_index == [0] diff --git a/tests/unit/tasks/test_lighteval_task.py b/tests/unit/tasks/test_lighteval_task.py deleted file mode 100644 index 7cdb7b6f5..000000000 --- a/tests/unit/tasks/test_lighteval_task.py +++ /dev/null @@ -1,86 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - - -from lighteval.tasks.lighteval_task import LightevalTask, LightevalTaskConfig -from lighteval.tasks.requests import Doc - - -def dummy_prompt_function(item, task_name): - return Doc(query=item["text"], choices=["A", "B"], gold_index=0, task_name=task_name) - - -def test_revision_check(): - # Test with a different revision - cfg_with_revision = LightevalTaskConfig( - name="test_task_revision", - prompt_function=dummy_prompt_function, - hf_repo="lighteval-tests-datasets/dataset-test-1", - hf_subset="default", - evaluation_splits=["train"], - metrics=[], - hf_revision="25175defadfde48b131b7cd7573ad6f59f868306", - ) - task_with_revision = LightevalTask(cfg_with_revision) - docs = task_with_revision.eval_docs() - queries = [doc.query for doc in docs] - assert queries == ["hi", "how are you?"] - - -def test_dataset_filter(): - # Setup - - cfg = LightevalTaskConfig( - name="test_task", - prompt_function=dummy_prompt_function, - hf_repo="lighteval-tests-datasets/dataset-test-1", - hf_subset="default", - hf_filter=lambda x: x["text"] == "hi", - metrics=[], - evaluation_splits=["train"], - ) - task = LightevalTask(cfg) - - filtered_docs = task.eval_docs() - assert len(filtered_docs) == 1 - assert filtered_docs[0].query == "hi" - - -def test_hf_data_files(tmp_path): - # create a small jsonl dataset - data_file = tmp_path / "data.jsonl" - src_docs = [f"document {i}" for i in range(3)] - data_file.write_text("\n".join([f'{{"text": "{doc}"}}' for doc in src_docs])) - - cfg = LightevalTaskConfig( - name="test_data_files", - prompt_function=dummy_prompt_function, - hf_repo="json", - hf_subset="default", - metrics=[], - evaluation_splits=["train"], - hf_data_files=str(data_file), - ) - task = LightevalTask(cfg) - - eval_docs = task.eval_docs() - assert [doc.query for doc in eval_docs] == src_docs diff --git a/tests/unit/tasks/test_registry.py b/tests/unit/tasks/test_registry.py deleted file mode 100644 index cc98cb212..000000000 --- a/tests/unit/tasks/test_registry.py +++ /dev/null @@ -1,122 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import pytest - -from lighteval.tasks.lighteval_task import LightevalTask, LightevalTaskConfig -from lighteval.tasks.registry import Registry - - -def test_superset_expansion(): - """ - Tests that task info selector correctly handles supersets. - """ - registry = Registry(tasks="storycloze|0") - - # The task list is saved as provided by the user - assert registry.tasks_list == ["storycloze|0"] - - # But we expand the superset when loading the configurations - assert set(registry.task_to_configs.keys()) == {"storycloze:2016", "storycloze:2018"} - - for task_name in {"storycloze:2016", "storycloze:2018"}: - task_info: list[LightevalTaskConfig] = registry.task_to_configs[task_name] - assert task_info[0].num_fewshots == 0 - - -def test_superset_with_subset_task(): - """ - Tests that task info selector correctly handles if both superset and one of subset tasks are provided. - """ - registry = Registry(tasks="mmlu|3,mmlu:abstract_algebra|5") - - # We have all mmlu tasks - assert set(registry.tasks_list) == {"mmlu|3", "mmlu:abstract_algebra|5"} - assert len(registry.task_to_configs.keys()) == 57 - - task_info: list[LightevalTaskConfig] = registry.task_to_configs["mmlu:abstract_algebra"] - assert {task_info[0].num_fewshots, task_info[1].num_fewshots} == {3, 5} - - -def test_cli_sampling_params(): - """ - Tests task setting the sampling parameters in CLI. - """ - registry_no_sampling = Registry(tasks="math_500|0") - - task_info_no_sampling: list[LightevalTaskConfig] = registry_no_sampling.task_to_configs["math_500"] - # Default values - assert task_info_no_sampling[0].metrics[0].sample_level_fn.k == 1 - assert task_info_no_sampling[0].metrics[0].sample_level_fn.n == 1 - - registry = Registry(tasks="math_500@k=2@n=10|0") - - task_info: list[LightevalTaskConfig] = registry.task_to_configs["math_500"] - assert task_info[0].metrics[0].sample_level_fn.k == 2 - assert task_info[0].metrics[0].sample_level_fn.n == 10 - - -def test_cli_sampling_params_fail(): - """ - Tests task setting the sampling parameters in CLI failure when args are wrong. - """ - # creation of object should fail - with pytest.raises(ValueError): - Registry("math_500@plop|0") - - -def test_task_group_expansion_with_subset_expansion(): - """ - Tests that task info selector correctly handles a group with task superset is provided. - """ - registry = Registry(tasks="mmlu|0") - - # We have all mmlu tasks - assert len(registry.task_to_configs.keys()) == 57 - - -def test_invalid_task_creation(): - """ - Tests that tasks info registry correctly raises errors for invalid tasks - """ - with pytest.raises(ValueError): - Registry(tasks="custom|task_revision") - - -def test_task_duplicates(): - """ - Tests that task info selector correctly handles if duplicate tasks are provided. - """ - registry = Registry(tasks="storycloze:2016|0,storycloze:2016|0") - - assert list(registry.tasks_list) == ["storycloze:2016|0"] - - -def test_task_creation(): - """ - Tests that tasks registry correctly creates tasks - """ - registry = Registry(tasks="storycloze:2016|0") - task = registry.load_tasks()["storycloze:2016|0"] - - assert isinstance(task, LightevalTask) - assert task.name == "storycloze:2016" diff --git a/tests/unit/test_unit_reorder.py b/tests/unit/test_unit_reorder.py deleted file mode 100644 index 1e01a1095..000000000 --- a/tests/unit/test_unit_reorder.py +++ /dev/null @@ -1,84 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from lighteval.data import GenerativeTaskDataset -from lighteval.metrics.utils.metric_utils import SamplingMethod -from lighteval.tasks.requests import Doc - - -# test data that will need to be sorted by length of the string -TEST_DATA = [ - Doc( - query="1 The quick brown fox jumps over the lazy dog", - choices=["A", "B", "C"], - gold_index=0, - ), - Doc( - query="2 The quick brown fox jumps over the lazy dog njsa", - choices=["A", "B", "C"], - gold_index=0, - ), - Doc( - query="Some text", - choices=["A", "B", "C"], - gold_index=0, - ), - Doc( - query="some more text", - choices=["A", "B", "C"], - gold_index=0, - ), - Doc( - query="not sure what to write here", - choices=["A", "B", "C"], - gold_index=0, - ), -] - -DATASET_SPLITS = 1 - - -class TestReorderGenerativeTaskDataset: - def test_reorder_dataset(self): - data = TEST_DATA.copy() - for d in data: - d.task_name = "test" - d.sampling_methods = [SamplingMethod.GENERATIVE] - d.generation_size = 10 - d.stop_sequences = ["stop", ":", "end"] - - dataset = GenerativeTaskDataset(requests=data, num_dataset_splits=DATASET_SPLITS) - - sorted_data = dataset.sorted_data - original_data = dataset.get_original_order(sorted_data) - - for i in range(len(sorted_data) - 1): - assert len(sorted_data[i].query) >= len(sorted_data[i + 1].query), ( - f"dataset[{i}][0] = {sorted_data[i].query} is shorter than dataset[{i + 1}][0] = {sorted_data[i + 1].query}" - ) - - assert len(sorted_data) == len(original_data), ( - f"reordered dataset has length {len(sorted_data)}, should be {len(dataset)}" - ) - - for sorted_data, orignal in zip(original_data, data): - assert sorted_data == orignal diff --git a/tests/unit/utils/test_caching.py b/tests/unit/utils/test_caching.py deleted file mode 100644 index 7ab8644be..000000000 --- a/tests/unit/utils/test_caching.py +++ /dev/null @@ -1,345 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import tempfile -import unittest -from dataclasses import asdict -from unittest.mock import Mock, patch - -import pytest -import torch - -from lighteval.models.abstract_model import LightevalModel -from lighteval.models.model_output import ModelResponse -from lighteval.tasks.requests import Doc, SamplingMethod -from lighteval.utils.cache_management import SampleCache -from lighteval.utils.imports import Extra, is_package_available - - -class TestCaching(unittest.TestCase): - def setUp(self): - """Create simple test documents.""" - self.docs = [] - self.model_responses = [] - self.task_name = "cache_test" - for i in range(3): - doc = Doc( - id=f"test_doc_{i}", - task_name=self.task_name, - query=f"Test question {i}: What is 2+2?", - choices=["3", "4", "5", "6"], - gold_index=1, - instruction="Answer the math question", - ) - model_resp = ModelResponse( - input=doc.query, - text=[f"Answer {i}"], - input_tokens=[1, 2, 3, 4], - output_tokens=[[5, 6, 7]], - logprobs=[-0.1, -0.2, -0.3], - argmax_logits_eq_gold=True, - ) - self.docs.append(doc) - self.model_responses.append(model_resp) - - def test_cache_directory_structure(self): - """Test that cache directories are created correctly.""" - from lighteval.models.dummy.dummy_model import DummyModelConfig - from lighteval.models.endpoints.endpoint_model import InferenceEndpointModelConfig - from lighteval.models.endpoints.tgi_model import TGIModelConfig - from lighteval.models.sglang.sglang_model import SGLangModelConfig - from lighteval.models.transformers.transformers_model import TransformersModelConfig - from lighteval.models.transformers.vlm_transformers_model import VLMTransformersModelConfig - from lighteval.models.vllm.vllm_model import VLLMModelConfig - - # We skip AdapterModelConfig, DeltaModelConfig because of imports - # We skip FullNanotronConfig as it's not standardized with our other configs, will need to be homogeneized - model_configs = [ - TransformersModelConfig, - VLMTransformersModelConfig, - VLLMModelConfig, - InferenceEndpointModelConfig, - TGIModelConfig, - SGLangModelConfig, - DummyModelConfig, - ] - - for model_config in model_configs: - with self.subTest(model_config=model_config): - with tempfile.TemporaryDirectory() as temp_dir: - model_name = f"test_model_{model_config.__name__}" - # if model_config in [AdapterModelConfig, DeltaModelConfig]: - # config = model_config(model_name=model_name, base_model=model_name + "2", cache_dir=temp_dir) - # else: - config = model_config(model_name=model_name, cache_dir=temp_dir) - - # Create cache with custom directory - cache = SampleCache(config) - - # Check directory structure - folder = cache.cache_dir - self.assertTrue(folder.exists()) - self.assertIn(str(temp_dir), str(folder)) - self.assertIn(model_name, str(folder)) - - def test_cache_decorator_presence(self): - """Test that @cached decorators are present on the right methods.""" - from lighteval.models.dummy.dummy_model import DummyModel - from lighteval.models.endpoints.endpoint_model import InferenceEndpointModel - from lighteval.models.endpoints.tgi_model import ModelClient - from lighteval.models.nanotron.nanotron_model import NanotronLightevalModel - from lighteval.models.sglang.sglang_model import SGLangModel - from lighteval.models.transformers.adapter_model import AdapterModel - from lighteval.models.transformers.delta_model import DeltaModel - from lighteval.models.transformers.transformers_model import TransformersModel - from lighteval.models.transformers.vlm_transformers_model import VLMTransformersModel - from lighteval.models.vllm.vllm_model import AsyncVLLMModel, VLLMModel - - model_classes = [ - TransformersModel, - AdapterModel, - DeltaModel, - VLMTransformersModel, - VLLMModel, - AsyncVLLMModel, - InferenceEndpointModel, - ModelClient, - NanotronLightevalModel, - SGLangModel, - DummyModel, - ] - methods_to_check = ["greedy_until", "loglikelihood", "loglikelihood_rolling"] - - for model_class in model_classes: - for method_name in methods_to_check: - with self.subTest(model_class=model_class, method_name=method_name): - self.assertTrue( - hasattr(model_class, method_name), f"{method_name} method not found for {model_class}" - ) - method = getattr(model_class, method_name) - # Check if method has been wrapped by @cached decorator - self.assertTrue( - hasattr(method, "__wrapped__"), f"{method_name} missing @cached decorator for {model_class}" - ) - - def _test_cache(self, model: LightevalModel, test_cases): - """Test that the @cached decorator logic works correctly - called by all model specific functions below.""" - for function_name, sampling_method in test_cases: - with self.subTest(function_name=function_name): - process_inputs = getattr(model, function_name) - process_inputs(self.docs) - - cache: SampleCache = model._cache - - # Check task_id - task_id = cache.get_task_id(self.task_name, sampling_method) - self.assertEqual(task_id.task_name, self.task_name) - self.assertEqual(task_id.sampling_method, sampling_method) - - # Verify cache files were created - cache_file = cache.get_cache_path(task_id) - self.assertTrue(cache_file.exists(), "Cache file not created") - - # Test retrieving from cache - self.assertEqual(cache._load_cached_indices()[task_id], [doc.id for doc in self.docs]) - uncached_docs, tasks_with_cached_samples = cache.get_samples_to_process_and_cache( - docs=self.docs, sampling_method=sampling_method - ) - self.assertEqual(tasks_with_cached_samples, {task_id}) - self.assertEqual( - len(uncached_docs), 0, f"{len(uncached_docs)} documents not found in cache when it should be 0" - ) - - # Verify cached results match original - cached_responses = cache.get_samples_from_cache( - docs=self.docs, task_ids=[task_id], sampling_method=sampling_method - ) - for cached_response, response in zip(cached_responses, self.model_responses): - self.assertEqual(asdict(cached_response), asdict(response)) - - @patch("lighteval.models.transformers.transformers_model.TransformersModel._loglikelihood_tokens") - @patch("lighteval.models.transformers.transformers_model.TransformersModel._padded_greedy_until") - @patch("lighteval.models.transformers.transformers_model.Accelerator") - @patch("lighteval.models.transformers.transformers_model.TransformersModel._create_auto_model") - def test_cache_transformers(self, mock_create_model, mock_accelerator, mock_greedy_until, mock_loglikelihood): - from lighteval.models.transformers.transformers_model import TransformersModel, TransformersModelConfig - - # Skip the model creation phase - mock_create_model = Mock() # noqa F841 - - # Mock accelerate related params - mock_accelerator_instance = Mock() - mock_accelerator_instance.device = torch.device("cpu") - mock_accelerator.return_value = mock_accelerator_instance - - mock_greedy_until.return_value = self.model_responses - mock_loglikelihood.return_value = self.model_responses - with tempfile.TemporaryDirectory() as temp_dir: - config = TransformersModelConfig(model_name="Qwen/Qwen3-0.6B", cache_dir=temp_dir) - model = TransformersModel(config) - - self._test_cache( - model, - [ - ("greedy_until", SamplingMethod.GENERATIVE), - ("loglikelihood", SamplingMethod.LOGPROBS), - ("loglikelihood_rolling", SamplingMethod.PERPLEXITY), - ], - ) - - @patch("lighteval.models.vllm.vllm_model.VLLMModel._loglikelihood_tokens") - @patch("lighteval.models.vllm.vllm_model.VLLMModel._greedy_until") - @patch("lighteval.models.vllm.vllm_model.VLLMModel._create_auto_model") - def test_cache_vllm(self, mock_create_model, mock_greedy_until, mock_loglikelihood): - from lighteval.models.vllm.vllm_model import VLLMModel, VLLMModelConfig - - # Mock VLLM LLM - mock_create_model = Mock() # noqa F841 - mock_greedy_until.return_value = self.model_responses - mock_loglikelihood.return_value = self.model_responses - - with tempfile.TemporaryDirectory() as temp_dir: - config = VLLMModelConfig(model_name="Qwen/Qwen3-0.6B", cache_dir=temp_dir) - model = VLLMModel(config) - - self._test_cache( - model, - [ - ("greedy_until", SamplingMethod.GENERATIVE), - ("loglikelihood", SamplingMethod.LOGPROBS), - ], - ) - - @patch("requests.get") - @patch("lighteval.models.endpoints.tgi_model.ModelClient._greedy_until") - @patch("lighteval.models.endpoints.tgi_model.ModelClient._loglikelihood") - def test_cache_tgi(self, mock_loglikelihood, mock_greedy_until, mock_requests_get): - from lighteval.models.endpoints.tgi_model import ModelClient, TGIModelConfig - - if not is_package_available(Extra.TGI): - pytest.skip("Skipping because missing the imports") - - # Mock TGI requests - mock_loglikelihood.return_value = self.model_responses - mock_greedy_until.return_value = self.model_responses - - # Mock HTTP info request - mock_requests_get.return_value.json.return_value = {"model_id": "Qwen/Qwen3-0.6B"} - - with tempfile.TemporaryDirectory() as temp_dir: - config = TGIModelConfig( - model_name="Qwen/Qwen3-0.6B", cache_dir=temp_dir, inference_server_address="http://localhost:8080" - ) - model = ModelClient(config) - - self._test_cache( - model, - [ - ("greedy_until", SamplingMethod.GENERATIVE), - ("loglikelihood", SamplingMethod.LOGPROBS), - ("loglikelihood_rolling", SamplingMethod.PERPLEXITY), - ], - ) - - @patch("lighteval.models.endpoints.endpoint_model.InferenceEndpointModel._loglikelihood") - @patch("lighteval.models.endpoints.endpoint_model.InferenceEndpointModel._greedy_until") - @patch("lighteval.models.endpoints.endpoint_model.InferenceEndpointModel._create_endpoint") - def test_cache_endpoint(self, mock_init, mock_greedy_until, mock_loglikelihood): - from lighteval.models.endpoints.endpoint_model import InferenceEndpointModel, InferenceEndpointModelConfig - - # Mock endpoint requests - auto_model = Mock() - auto_model.repository = "Qwen/Qwen3-0.6B" - auto_model.revision = "" - mock_init.return_value = auto_model, Mock(), Mock() # noqa F841 - - mock_greedy_until.return_value = self.model_responses - mock_loglikelihood.return_value = self.model_responses - - with tempfile.TemporaryDirectory() as temp_dir: - config = InferenceEndpointModelConfig(model_name="Qwen/Qwen3-0.6B", cache_dir=temp_dir) - model = InferenceEndpointModel(config) - - self._test_cache( - model, - [ - ("greedy_until", SamplingMethod.GENERATIVE), - ("loglikelihood", SamplingMethod.LOGPROBS), - ("loglikelihood_rolling", SamplingMethod.PERPLEXITY), - ], - ) - - @patch("lighteval.models.sglang.sglang_model.SGLangModel._loglikelihood_tokens") - @patch("lighteval.models.sglang.sglang_model.SGLangModel._greedy_until") - @patch("lighteval.models.sglang.sglang_model.SGLangModel._create_auto_tokenizer") - @patch("lighteval.models.sglang.sglang_model.SGLangModel._create_auto_model") - def test_cache_sglang( - self, mock_create_auto_model, mock_create_auto_tokenizer, mock_greedy_until, mock_loglikelihood - ): - from lighteval.models.sglang.sglang_model import SGLangModel, SGLangModelConfig - - # Mock SGLang engine - mock_create_auto_model = Mock() # noqa F841 - mock_create_auto_tokenizer = Mock() # noqa F841 - mock_greedy_until.return_value = self.model_responses - mock_loglikelihood.return_value = self.model_responses - - with tempfile.TemporaryDirectory() as temp_dir: - config = SGLangModelConfig(model_name="Qwen/Qwen3-0.6B", cache_dir=temp_dir) - model = SGLangModel(config) - - self._test_cache( - model, - [ - ("greedy_until", SamplingMethod.GENERATIVE), - ("loglikelihood", SamplingMethod.LOGPROBS), - ], - ) - - @patch("lighteval.models.transformers.vlm_transformers_model.VLMTransformersModel._greedy_until") - @patch("lighteval.models.transformers.vlm_transformers_model.Accelerator") - @patch("lighteval.models.transformers.vlm_transformers_model.VLMTransformersModel._create_auto_model") - def test_cache_vlm_transformers(self, mock_create_model, mock_accelerator, mock_greedy_until): - from lighteval.models.transformers.vlm_transformers_model import ( - VLMTransformersModel, - VLMTransformersModelConfig, - ) - - # Mock accelerate related params - mock_accelerator_instance = Mock() - mock_accelerator_instance.device = torch.device("cpu") - mock_accelerator.return_value = mock_accelerator_instance - - # Skip the model creation phase - mock_create_model = Mock() # noqa F841 - mock_greedy_until.return_value = self.model_responses - - with tempfile.TemporaryDirectory() as temp_dir: - config = VLMTransformersModelConfig(model_name="HuggingFaceTB/SmolVLM-256M-Instruct", cache_dir=temp_dir) - model = VLMTransformersModel(config) - - self._test_cache( - model, - [ - ("greedy_until", SamplingMethod.GENERATIVE), - ], - ) diff --git a/tests/unit/utils/test_utils.py b/tests/unit/utils/test_utils.py deleted file mode 100644 index 68e844712..000000000 --- a/tests/unit/utils/test_utils.py +++ /dev/null @@ -1,63 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -import unittest - -from lighteval.utils.utils import remove_reasoning_tags - - -class TestRemoveReasoningTags(unittest.TestCase): - def test_remove_reasoning_tags(self): - text = " Reasoning section Answer section" - tag_pairs = [("", "")] - result = remove_reasoning_tags(text, tag_pairs) - self.assertEqual(result, " Answer section") - - def test_remove_multiple_tags(self): - text = " Reasoning Interlude More reasoning Answer" - tag_pairs = [("", "")] - result = remove_reasoning_tags(text, tag_pairs) - self.assertEqual(result, " Interlude Answer") - - def test_no_tags(self): - text = "No reasoning tags here." - tag_pairs = [("", "")] - result = remove_reasoning_tags(text, tag_pairs) - self.assertEqual(result, "No reasoning tags here.") - - def test_empty_text(self): - text = "" - tag_pairs = [("", "")] - result = remove_reasoning_tags(text, tag_pairs) - self.assertEqual(result, "") - - def test_no_opening_tag(self): - text = "No opening tag Reasoning section. Answer section" - tag_pairs = [("", "")] - result = remove_reasoning_tags(text, tag_pairs) - self.assertEqual(result, "No opening tag Answer section") - - def test_no_closing_tag(self): - text = " Reasoning section. Answer section" - tag_pairs = [("", "")] - result = remove_reasoning_tags(text, tag_pairs) - self.assertEqual(result, " Reasoning section. Answer section") diff --git a/tests/utils.py b/tests/utils.py deleted file mode 100644 index b8c71c76c..000000000 --- a/tests/utils.py +++ /dev/null @@ -1,135 +0,0 @@ -# MIT License - -# Copyright (c) 2024 The HuggingFace Team - -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. - -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -from typing import Optional -from unittest.mock import patch - -from transformers import AutoTokenizer - -from lighteval.logging.evaluation_tracker import EvaluationTracker -from lighteval.models.abstract_model import LightevalModel, ModelConfig -from lighteval.models.model_output import ModelResponse -from lighteval.pipeline import ParallelismManager, Pipeline, PipelineParameters -from lighteval.tasks.lighteval_task import LightevalTask -from lighteval.tasks.registry import Registry -from lighteval.tasks.requests import Doc -from lighteval.utils.cache_management import SampleCache -from lighteval.utils.imports import is_package_available - - -class FakeModelConfig(ModelConfig): - model_name: str = "fake_model" - - -class FakeModel(LightevalModel): - """Fake model for testing purposes.""" - - def __init__( - self, - greedy_until_responses: list[ModelResponse] = [], - loglikelihood_responses: list[ModelResponse] = [], - loglikelihood_rolling_responses: list[ModelResponse] = [], - ): - self._tokenizer = None - self.config = FakeModelConfig() - self.greedy_until_responses = greedy_until_responses - self.loglikelihood_responses = loglikelihood_responses - self.loglikelihood_rolling_responses = loglikelihood_rolling_responses - self._cache = SampleCache(self.config) - - @property - def tokenizer(self): - if self._tokenizer is None: - self._tokenizer = AutoTokenizer.from_pretrained("gpt2") - return self._tokenizer - - @property - def add_special_tokens(self): - return False - - @property - def max_length(self) -> int: - return 2048 - - def greedy_until(self, docs: list[Doc]) -> list[ModelResponse]: - ret_resp, self.greedy_until_responses = ( - self.greedy_until_responses[: len(docs)], - self.greedy_until_responses[len(docs) :], - ) - return ret_resp - - def loglikelihood(self, docs: list[Doc]) -> list[ModelResponse]: - ret_resp, self.loglikelihood_responses = ( - self.loglikelihood_responses[: len(docs)], - self.loglikelihood_responses[len(docs) :], - ) - return ret_resp - - def loglikelihood_rolling(self, docs: list[Doc]) -> list[ModelResponse]: - ret_resp, self.loglikelihood_rolling_responses = ( - self.loglikelihood_rolling_responses[: len(docs)], - self.loglikelihood_rolling_responses[len(docs) :], - ) - return ret_resp - - -def fake_evaluate_task( - lighteval_task: LightevalTask, lm: FakeModel, max_samples: int = 1, n_fewshot: int = 0, n_fewshot_seeds: int = 1 -): - # Mock the Registry.get_task_dict method - - task_name = f"{lighteval_task.name}" - task_name_fs = f"{lighteval_task.name}|{n_fewshot}" - - task_dict = {task_name: lighteval_task} - evaluation_tracker = EvaluationTracker(output_dir="outputs") - evaluation_tracker.task_config_logger.log(task_dict) - # Create a mock Registry class - - class FakeRegistry(Registry): - def __init__(self, tasks: Optional[str], load_multilingual: bool = False, custom_tasks: Optional[str] = None): - self.tasks_list = [task_name_fs] - self.task_to_configs = {task_name_fs: [lighteval_task.config]} - - def load_tasks(self): - return {task_name_fs: lighteval_task} - - # This is due to logger complaining we have no initialised the accelerator - # It's hard to mock as it's global singleton - if is_package_available("accelerate"): - from accelerate import Accelerator - - Accelerator() - - # This is a bit hacky, because there is no way to run end to end, with - # dynamic task :(, so we just mock the registry - with patch("lighteval.pipeline.Registry", FakeRegistry): - pipeline = Pipeline( - tasks=task_name, - pipeline_parameters=PipelineParameters(max_samples=max_samples, launcher_type=ParallelismManager.NONE), - evaluation_tracker=evaluation_tracker, - model=lm, - model_config=None, - ) - pipeline.evaluate() - - return pipeline.get_results()