update_benchmark.py

import runloop_api_client

client = runloop_api_client.Runloop()

BENCHMARK_METADATA = {
    'princeton-nlp/SWE-bench_Verified': {
        'id': 'bmd_2zfI4J2JrAliAdy04re0y',
        'description':
        "SWE-bench Verified is a high-quality, human-validated subset of the original SWE-bench. It consists of 500 samples that software engineers have confirmed are solvable and non-problematic, addressing issues like underspecified problem statements or unreliable development environment setups found in the original dataset. This version is intended to provide a more accurate and reliable measure of AI models' abilities to solve real-world software issues.",
        'attribution':
        'Released as a collaboration between the original SWE-bench authors and OpenAI Preparedness.',
    },
    'ByteDance-Seed/Multi-SWE-bench': {
        'id': 'bmd_2zpUcMGXJ8s3pr8RaAvEg',
        'description':
        'Multi-SWE-Bench is a benchmark designed to evaluate the issue-resolving capabilities of Large Language Models (LLMs) across multiple programming languages. It expands on the concept of SWE-bench by including tasks in seven languages: Java, TypeScript, JavaScript, Go, Rust, C, and C++. The dataset comprises 1,632 high-quality instances curated by expert annotators.',
        'attribution':
        'Daoguang Zan, Zhirong Huang, Wei Liu, Hanwu Chen, Linhao Zhang, Shulin Xin, Lu Chen, Qi Liu, Xiaojian Zhong, Aoyan Li, Siyao Liu, Yongsheng Xiao, Liangqiang Chen, and Yuyu Zhang. (Associated with ByteDance-Seed)',
    },
    'SWE-bench/SWE-smith': {
        'id': 'bmd_303o8ec5wVATnovx6eVfx',
        'description':
        'SWE-smith is a toolkit designed to generate a large number of SWE-bench style task instances for any given Python repository. It allows users to create training data for software engineering agents, generate trajectories of these agents solving the tasks, and train local language models on these trajectories to enhance their software engineering capabilities.',
        'attribution':
        'John Yang, Kilian Leret, Carlos E. Jimenez, Alexander Wettig, Kabir Khandpur, Yanzhe Zhang, Binyuan Hui, Ofir Press, Ludwig Schmidt, and Diyi Yang.',
    },
    'openai_humaneval': {
        'id': 'bmd_30IZMpToR9lGtGtXXBssi',
        'description':
        "HumanEval is a benchmark designed to evaluate the functional correctness of code generated by large language models. It consists of 164 hand-written programming problems in Python, each with a docstring-based prompt and a hidden unit test suite. HumanEval measures a model's ability to synthesize correct solutions with no prior access to test cases, making it a standard benchmark for zero-shot code generation.",
        'attribution':
        'Mark Chen, Jerry Tworek, Heewoo Jun, Qiming Yuan, Henrique Ponde de Oliveira Pinto, Jared Kaplan, Harri Edwards, Yuri Burda, Nicholas Joseph, Greg Brockman, Alex Ray, Raul Puri, Gretchen Krueger, Michael Petrov, Heidy Khlaaf, Girish Sastry, Pamela Mishkin, Brooke Chan, Scott Gray, and Ilya Sutskever.',
    },
    'OpenAI/SWE-Lancer': {
        'id': 'bmd_30nvsaEGemfJdwnAMDeAM',
        'description':
        'SWE-Lancer is a large-scale benchmark of 1,488 real freelance software engineering tasks sourced from Upwork, totaling $1M in contracts. It evaluates models on two fronts: solving tasks as Individual Contributors (IC) using end-to-end test cases, and selecting top freelancer submissions as SWE Managers. A public "Diamond" split enables open benchmarking, while earnings-based scoring quantifies real-world utility.',
        'attribution':
        'Samuel Miserendino, Michele Wang, Tejal Patwardhan, Johannes Heidecke (all from OpenAI)',
    },
}

for name, benchmark in BENCHMARK_METADATA.items():
    client.benchmarks.update(benchmark['id'], name=name, attribution=benchmark['attribution'], description=benchmark['description'])
    # print(benchmark['id'], name, benchmark['attribution'], benchmark['description'])