修复算子导入脚本

Author: hhhhsc701

tools/utils/load_operators.sh

@@ -110,20 +110,29 @@ read -r -d '' PY_SCRIPT << 'EOF'
 from pathlib import Path
 import json, sys, yaml, base64
 
-operator_sql = 'INSERT IGNORE INTO t_operator (id, name, description, version, inputs, outputs, runtime, settings, file_name, is_star) VALUES '
-category_sql = 'INSERT IGNORE INTO t_operator_category_relation(category_id, operator_id) VALUES '
+operator_sql = 'INSERT INTO t_operator (id, name, description, version, inputs, outputs, runtime, settings, file_name, is_star, metrics, file_size) VALUES '
+category_sql = 'INSERT INTO t_operator_category_relation(category_id, operator_id) VALUES '
+release_sql = 'INSERT INTO t_operator_release (id, version, release_date, changelog) VALUES '
+
+# Modal类型映射 (与前端分类ID对应)
 modal_map = {
     'text': 'd8a5df7a-52a9-42c2-83c4-01062e60f597',
     'image': 'de36b61c-9e8a-4422-8c31-d30585c7100f',
     'audio': '42dd9392-73e4-458c-81ff-41751ada47b5',
     'video': 'a233d584-73c8-4188-ad5d-8f7c8dda9c27',
     'multimodal': '4d7dbd77-0a92-44f3-9056-2cd62d4a71e4'
 }
-language_map = {
-    'python': '9eda9d5d-072b-499b-916c-797a0a8750e1',
-    'java': 'b5bfc548-8ef6-417c-b8a6-a4197c078249'
+
+function_map = {
+    'cleaning': '8c09476a-a922-418f-a908-733f8a0de521',
+    'annotation': 'cfa9d8e2-5b5f-4f1e-9f12-1234567890ab'
 }
 
+# 系统预置分类ID
+PREDEFINED_ID = 'ec2cdd17-8b93-4a81-88c4-ac9e98d10757'
+VENDOR_CATEGORY_ID = 'f00eaa3e-96c1-4de4-96cd-9848ef5429ec'
+PYTHON_CATEGORY_ID = '9eda9d5d-072b-499b-916c-797a0a8750e1'
+
 base_path = Path('/opt/runtime/datamate/ops/user')
 for metadata_file in base_path.rglob('metadata.yml'):
     try:
@@ -140,25 +149,66 @@ for metadata_file in base_path.rglob('metadata.yml'):
 
             runtime = f"'{json.dumps(data.get('runtime'), ensure_ascii=False)}'" if 'runtime' in data else 'null'
             settings = f"'{json.dumps(data.get('settings'), ensure_ascii=False)}'" if 'settings' in data else 'null'
-            file_name = Path(metadata_file).parent.name
+            metrics = f"'{json.dumps(data.get('metrics', []), ensure_ascii=False)}'" if 'metrics' in data else 'null'
+
+            # 计算文件大小
+            root_dir = Path(metadata_file).parent
+            file_size = sum(f.stat().st_size for f in root_dir.rglob('*') if f.is_file())
+
+            file_name = root_dir.name
+
+            # 构建operator插入SQL
+            operator_sql += "('{}', '{}', '{}', '{}', '{}', '{}', {}, {}, '{}', 'false', {}, {}),".format(
+                id, name, desc, version, inputs, outputs, runtime, settings, file_name, metrics, file_size
+            )
+
+            # 功能类型分类（支持多个types）
+            types = data.get('types', [])
+            has_type = False
+            if isinstance(types, list):
+                for func_type in types:
+                    func_category_id = function_map.get(func_type)
+                    if func_category_id:
+                        has_type = True
+                        category_sql += f"('{func_category_id}', '{id}'),"
+            if not has_type:
+                category_sql += f"('{function_map.get('cleaning')}', '{id}'),"
+
+            # 系统预置分类
+            category_sql += f"('{PREDEFINED_ID}', '{id}'),"
+            category_sql += f"('{VENDOR_CATEGORY_ID}', '{id}'),"
+            category_sql += f"('{PYTHON_CATEGORY_ID}', '{id}'),"
+            category_sql += f"('{modal_map.get(modal, modal_map.get('text'))}', '{id}'),"
+
+            # Release信息
+            releases = data.get('release', [])
+            if isinstance(releases, list) and releases:
+                release_sql += f"('{id}', '{version}', NOW(), '{json.dumps(releases, ensure_ascii=False)}'),"
 
-            operator_sql += "('{}', '{}', '{}', '{}', '{}', '{}', {}, {}, '{}', 'false'),".format(id, name, desc, version, inputs, outputs, runtime, settings, file_name)
-            category_sql += "('{}', '{}'),".format(modal_map.get(modal, 'd8a5df7a-52a9-42c2-83c4-01062e60f597'), id)
-            category_sql += "('{}', '{}'),".format(language_map.get(language, '9eda9d5d-072b-499b-916c-797a0a8750e1'), id)
-            category_sql += "('ec2cdd17-8b93-4a81-88c4-ac9e98d10757', '{}'),".format(id)
-            category_sql += "('f00eaa3e-96c1-4de4-96cd-9848ef5429ec', '{}'),".format(id)
     except Exception as e:
         print(f'ERROR: {e}', file=sys.stderr)
         sys.exit(1)
-print(operator_sql[:-1] + ';\n' + category_sql[:-1] + ';')
+
+# 输出SQL
+full_sql = ''
+if operator_sql != 'INSERT INTO t_operator (id, name, description, version, inputs, outputs, runtime, settings, file_name, is_star, metrics, file_size) VALUES ':
+    full_sql += operator_sql[:-1] + ' ON CONFLICT DO NOTHING;\n'
+
+if category_sql != 'INSERT INTO t_operator_category_relation(category_id, operator_id) VALUES ':
+    full_sql += category_sql[:-1] + ' ON CONFLICT DO NOTHING;\n'
+
+if release_sql != 'INSERT INTO t_operator_release (id, version, release_date, changelog) VALUES ':
+    full_sql += release_sql[:-1] + ' ON CONFLICT DO NOTHING;\n'
+
+print(full_sql)
 EOF
 
 B64_CODE=$(python3 -c "import base64, sys; print(base64.b64encode(sys.stdin.read().encode('utf-8')).decode('utf-8'))" <<< "$PY_SCRIPT")
 FULL_SQL=$(kubectl exec -i "$HEAD_POD_NAME" -n "$NAMESPACE" -c ray-head -- /bin/sh -c "echo '$B64_CODE' | base64 -d | python3 -")
 
 log_info "插入数据库..."
 DATABASE_POD_NAME=$(kubectl get pod -n "$NAMESPACE" -l app.kubernetes.io/name=datamate-database -o jsonpath='{.items[*].metadata.name}')
-kubectl exec -i "$DATABASE_POD_NAME" -n "$NAMESPACE" -c database -- sh -c 'MYSQL_PWD="$MYSQL_ROOT_PASSWORD" mysql -uroot datamate' << EOF
+kubectl exec -i "$DATABASE_POD_NAME" -n "$NAMESPACE" -c database -- sh -c 'PGPASSWORD="$POSTGRES_PASSWORD" psql -U "$POSTGRES_USER" -d datamate' << EOF
 $FULL_SQL
 EOF