diff --git a/.github/workflows/pr-auto-label.yml b/.github/workflows/pr-auto-label.yml index 6bf97b51..90962791 100644 --- a/.github/workflows/pr-auto-label.yml +++ b/.github/workflows/pr-auto-label.yml @@ -37,7 +37,8 @@ jobs: 'dependabot': 'dependencies', 'upgrade': 'dependencies', 'security': 'security', - 'ci': 'ci' + 'ci': 'ci', + 'refactor': 'maintenance' }; // 提取标题的第一个单词(冒号前的部分) diff --git a/evaluate/base/evaluation.py b/evaluate/base/evaluation.py index cfe3e058..c81a9367 100644 --- a/evaluate/base/evaluation.py +++ b/evaluate/base/evaluation.py @@ -4,6 +4,7 @@ import re import time import uuid +from typing import Dict, Any, List from bohrium import Bohrium from dotenv import load_dotenv, find_dotenv @@ -115,13 +116,6 @@ def multi_turn_evaluation_task(dataset_item): return result -import asyncio -import json -import re -import time -from typing import Dict, Any, List - - async def _run_conversation(dataset_item: Dict[str, Any], max_turn_count: int, save_mode: str = 'w') -> Dict[str, Any]: """ 执行一次对话测试,并返回结果 diff --git a/evaluate/experiments/threads/database_search/run.sh b/evaluate/experiments/threads/run.sh similarity index 52% rename from evaluate/experiments/threads/database_search/run.sh rename to evaluate/experiments/threads/run.sh index a6572de6..b1620f78 100755 --- a/evaluate/experiments/threads/database_search/run.sh +++ b/evaluate/experiments/threads/run.sh @@ -1,16 +1,26 @@ #!/bin/bash -PYTHON=.venv/bin/python # your .venv + +SCRIPT_PATH=$(realpath "$BASH_SOURCE") +MATMASTER_DIR=$(dirname $(dirname $(dirname $(dirname "$SCRIPT_PATH")))) +THREADS_DIR=$MATMASTER_DIR/evaluate/experiments/threads + +if [ x"$1" == x ];then + echo "Please specify evaluation type...[$(ls -l $THREADS_DIR | grep '^d' | awk '{print $9}'| xargs)]" + exit 1 +fi + +PYTHON=$MATMASTER_DIR/.venv/bin/python # your .venv set -a -source .env # your .env +source $MATMASTER_DIR/.env # your .env set +a -export PYTHONPATH=/your/matmaster/path/MatMaster:$PYTHONPATH +export PYTHONPATH=$MATMASTER_DIR:$PYTHONPATH export MAX_JOBS=3 TOTAL=$($PYTHON -c " import os import json -with open('database_search.json') as f: +with open('$THREADS_DIR/$1/$1.json') as f: dataset_json = json.load(f) print(len(dataset_json)) ") @@ -22,7 +32,7 @@ running_jobs=0 for ((i=0; i<$TOTAL; i++)); do echo "🚀 提交任务: item $i" sleep 3 - $PYTHON database_search_bash.py \ + $PYTHON $THREADS_DIR/$1/$1_bash.py \ --item_id $i > item_$i.log 2>&1 & ((running_jobs++)) diff --git a/evaluate/experiments/threads/structure_generate/run.sh b/evaluate/experiments/threads/structure_generate/run.sh deleted file mode 100755 index 7f5e39bb..00000000 --- a/evaluate/experiments/threads/structure_generate/run.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -PYTHON=.venv/bin/python # your .venv -set -a -source .env # your .env -set +a - -export PYTHONPATH=/your/matmaster/path/MatMaster:$PYTHONPATH -export MAX_JOBS=3 - -TOTAL=$($PYTHON -c " -import os -import json -with open('structure_generate.json') as f: - dataset_json = json.load(f) -print(len(dataset_json)) -") - -echo '总数据量:' $TOTAL - -running_jobs=0 - -for ((i=0; i<$TOTAL; i++)); do - echo "🚀 提交任务: item $i" - sleep 3 - $PYTHON structure_generate_bash.py \ - --item_id $i > item_$i.log 2>&1 & - - ((running_jobs++)) - - # 如果正在运行的任务数达到上限,就等待任意一个完成 - if (( running_jobs >= MAX_JOBS )); then - wait -n - ((running_jobs--)) - fi -done - -# 等待最后一批任务 -wait -echo "✅ 所有任务完成"